{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "089abfb7",
   "metadata": {},
   "source": [
    "# Building Blocks of PyTorch"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8fd7272",
   "metadata": {},
   "source": [
    "#### System Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "dd05ac7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "18be7474",
   "metadata": {},
   "source": [
    "#### PyTorch Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e8fad9a9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/atifadib/opt/anaconda3/envs/torch_env/lib/python3.10/site-packages/transformers/utils/generic.py:441: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.\n",
      "  _torch_pytree._register_pytree_node(\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torchvision.io import read_image\n",
    "from torch.utils.data import Dataset\n",
    "from torchvision import transforms as T\n",
    "from torch.nn import functional as F \n",
    "from torch import optim\n",
    "from torch.nn import functional as F"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a0e963af",
   "metadata": {},
   "source": [
    "#### Loading MNIST Dataset using Custom Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fcb7dabc",
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomMNISTDataset(Dataset):\n",
    "\tdef __init__(self, dataset_path):\n",
    "\t\tself.dataset_path = dataset_path\n",
    "\t\tself.imgs, self.labels = [], []\n",
    "\t\tfor dirpath, _, filenames in os.walk(self.dataset_path):\n",
    "\t\t\tself.imgs.extend([f\"{dirpath}/{i}\" for i in filenames])\n",
    "\t\t\tself.labels.extend([f\"{dirpath.split('/')[-1]}\" for _ in filenames])\n",
    "\t\t\tself.transform = T.Compose([T.Lambda(torch.flatten)])\n",
    "\t\t\tself.target_transform = T.Compose([T.Lambda(lambda x: torch.LongTensor([x])), T.Lambda(lambda x: F.one_hot(x, 10))])\n",
    "\n",
    "\tdef __len__(self):\n",
    "\t\t# Returns the Size of the Dataset\n",
    "\t\treturn len(self.imgs)\n",
    "\n",
    "\tdef __getitem__(self, idx):\n",
    "\t\t# Returns a Sample from the data\n",
    "\t\timage = read_image(self.imgs[idx])\n",
    "\t\tlabel = int(self.labels[idx]) # Converting String Label to Integer\n",
    "\t\timage = self.transform(image)\n",
    "\t\timage = image / 255.0\n",
    "\t\tlabel = self.target_transform(label)\n",
    "\t\treturn image, label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "efcd3d43",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "\n",
    "train_dataloader = DataLoader(CustomMNISTDataset('../chapter3/output/'),\n",
    "                              batch_size=64, shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8df7af0c",
   "metadata": {},
   "source": [
    "#### Single Sample from MNIST Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3873841a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfsAAAGzCAYAAAAogL7TAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA/BUlEQVR4nO3deVxU9f4/8NewzAw7gqyCqJiZiFqEuORO4HJNyyXNCs1cEu2q2WKZaNYltWt21SittAhbLM029aum5jWXtMy8milhboCIMcMi63x+f/hjcphBOUdg4OPr+XjM4wFnzvsznzlzZl5zzplzPhohhAARERFJy8HeHSAiIqK6xbAnIiKSHMOeiIhIcgx7IiIiyTHsiYiIJMewJyIikhzDnoiISHIMeyIiIskx7ImIiCR3S4W9RqPBvHnz7N0NqmLs2LFwd3ev1TZ79+6N3r1712qbt4pPP/0UPj4+KCgoAACcPn0aGo3GfPvss8/M844dO9Y8vX379vbqMlGjc/jw4WrfV0OHDrX5vjp27BicnJxw9OhRxY+nOOx//fVXDB8+HGFhYdDr9WjWrBnuvfdeLFu2TPGDN3YtWrTAP/7xD3t3wy5u5ecOXH3TzZs3D6dPn7Z3V2pVRUUFkpKSMG3aNKsvYBMnTkRqaio6d+5sMb1p06ZITU3Fq6++atXeDz/8gHvuuQeurq4IDAzEk08+af4Sodbx48fRv39/uLu7w8fHB4888ghycnJuqs3z589j5MiR8Pb2hqenJ4YMGYI//vjjptrMy8vDxIkT4efnBzc3N/Tp0wc//fTTTbVZUlKCZ599FsHBwXBxcUFMTAy2bt16U22aTCYsWrQILVu2hF6vR4cOHfDRRx/dVJsA8O677+KOO+6AXq/HbbfdVisZ8eWXX+Kuu+6CXq9H8+bNkZSUhPLy8ptqs7bX0QMHDmDKlCmIioqCs7MzNBqNzfnCwsKQmpqK559/3uq+GTNmIDU1FW3btrWY3q5dOwwaNAhz585V3jGhwJ49e4RWqxWtW7cWCxYsEKtWrRJz584VcXFxIjw8XElTdgFAJCUl1Vp7YWFhYtCgQbXWXmNSm889ISFBuLm51UpblXr16iV69epVq21ea926dQKA2LFjR509hj1s2LBBaDQace7cOfO0jIwMAUCsXr3aav6EhAQRFhZms62ff/5Z6PV6ceedd4qUlBTxwgsvCJ1OJ/r376+6f2fPnhVNmzYV4eHh4o033hCvvPKKaNKkiejYsaMoKSlR1WZ+fr647bbbhL+/v1i4cKFYsmSJCA0NFSEhIeLSpUuq2qyoqBDdunUTbm5uYt68eWL58uWiXbt2wsPDQ/z++++q2hRCiFGjRgknJycxa9Ys8fbbb4uuXbsKJycnsXv3btVtPvfccwKAmDBhgli5cqUYNGiQACA++ugj1W2+9dZbAoAYNmyYWLlypXjkkUcEAPHqq6+qbvPbb78VGo1G9OnTR6xcuVJMmzZNODg4iMmTJ6tusy7W0aSkJOHs7CyioqJEmzZtxI1idseOHQKAWLdundV9vXr1EhERERbTvv32WwFAnDp1SlG/FIX9wIEDhZ+fn/jrr7+s7svOzlb0wPbAsK89soR9QUGBSE9PV9y+TGF/5coVUVFRIYQQ4r777hP33HOPxf1qw37AgAEiKChIGAwG87RVq1YJAGLLli2q+vrEE08IFxcX8eeff5qnbd26VQAQb7/9tqo2Fy5cKACIAwcOmKcdP35cODo6itmzZ6tq85NPPrH6AL948aLw9vYWo0ePVtXm/v37BQCxePFi87QrV66I8PBw0bVrV1Vtnjt3Tjg7O4vExETzNJPJJHr06CFCQkJEeXm54jaLioqEr6+v1efDmDFjhJubm7h8+bKqvrZr10507NhRlJWVmae98MILQqPRiOPHj6tqsy7W0aysLFFUVCSEECIxMbHWw760tFQ0adJEvPjii4r6pWg3fnp6OiIiIuDt7W11n7+/v8X/q1evRt++feHv7w+dTod27dohJSXFqq5yd/DOnTtx9913w8XFBZGRkdi5cycAYP369YiMjIRer0dUVBR+/vlni/rK471//PEH4uPj4ebmhuDgYLz00ksQNRjQ7/z583jssccQEBAAnU6HiIgIvPfeezVfKNeoPLb52muvYcWKFWjVqhVcXV0RFxeHs2fPQgiBBQsWICQkBC4uLhgyZAguX75s0cbGjRsxaNAgBAcHQ6fTITw8HAsWLEBFRYXV41U+houLCzp37ozdu3fbPFZdUlKCpKQktG7dGjqdDqGhoXjmmWdQUlJiMd+lS5fw22+/oaioSNXzr2r37t0YMWIEmjdvbn7cGTNm4MqVKzbnr8lraDKZsHTpUkRERECv1yMgIACTJk3CX3/9paqPOTk5aN26Nfr27Yu1a9eiuLj4hjVr1qzBiBEjAAB9+vQxH1urXGcBYNOmTejRowfc3Nzg4eGBQYMG4X//+59FO5Xr7vnz5zF06FC4u7vDz88Ps2bNsnq9P/74Y0RFRcHDwwOenp6IjIzEG2+8YTHPH3/8gREjRsDHxweurq7o0qULvvnmG4t5du7cCY1Gg48//hhz5sxBs2bN4OrqCqPRiOLiYmzevBmxsbFKFqFNRqMRW7duxcMPPwxPT0/z9EcffRTu7u749NNPVbX7+eef4x//+AeaN29unhYbG4s2bdqobvOzzz5DdHQ0oqOjzdPatm2Lfv363VSbAQEBeOCBB8zT/Pz8MHLkSGzcuNHqvVfTNh0dHTFx4kTzNL1ej/Hjx2Pv3r04e/as4jY3btyIsrIyTJkyxTxNo9HgiSeewLlz57B3717Fbe7YsQO5ubkWbQJAYmIiCgsLrdbJmjh27BiOHTuGiRMnwsnJyTx9ypQpEEJYHO+uqbpaRwMCAuDi4qKqtiacnZ3Ru3dvbNy4UVGdorAPCwvDoUOHavTjgJSUFISFheH555/Hv//9b4SGhmLKlClYsWKF1bynTp3CQw89hMGDByM5ORl//fUXBg8ejLS0NMyYMQMPP/ww5s+fj/T0dIwcORImk8mivqKiAv3790dAQAAWLVqEqKgoJCUlISkp6bp9zM7ORpcuXbBt2zZMnToVb7zxBlq3bo3x48dj6dKlShaNhbS0NLz55puYNm0annrqKezatQsjR47EnDlzsHnzZjz77LOYOHEivvrqK8yaNcuids2aNXB3d8fMmTPxxhtvICoqCnPnzsVzzz1nMV9KSgqmTp2KkJAQLFq0CD169MDQoUNx7tw5i/lMJhPuu+8+vPbaaxg8eDCWLVuGoUOH4vXXX8eDDz5oMe/y5ctxxx134MCBA6qf+7XWrVuHoqIiPPHEE1i2bBni4+OxbNkyPProo1bz1vQ1nDRpEp5++ml0794db7zxBsaNG4e0tDTEx8ejrKxMcR+DgoLw2muvIScnB2PGjEFQUBCmTp1q9aXyWj179sSTTz4JAHj++eeRmpqK1NRU3HHHHQCA1NRUDBo0CO7u7li4cCFefPFFHDt2DPfcc4/VMf6KigrEx8fD19cXr732Gnr16oV///vfWLlypXmerVu3YvTo0WjSpAkWLlyIV199Fb1798aePXvM82RnZ6Nbt27YsmULpkyZgldeeQXFxcW47777sGHDBqvnsGDBAnzzzTeYNWsW/vWvf0Gr1eLQoUMoLS3FXXfdpXg5VvXrr7+ivLwcd999t8V0rVaLTp06XXf5Vuf8+fO4ePGiVZsA0LlzZ1VtmkwmHDlypNo209PTkZ+fr7jdn3/+GXfddRccHCw/Yjt37oyioiL8/vvvqtps06aNRTBVtglc/cGXmjbd3NzM627VNtUs08qaqss0KioKDg4OtdpmcHAwQkJCVLVZF+tofYmKisLRo0dhNBprXqRkN8D//d//CUdHR+Ho6Ci6du0qnnnmGbFlyxZRWlpqNW/lboxrxcfHi1atWllMCwsLEwDEDz/8YJ62ZcsWAcBqd93bb79ttes0ISFBABDTpk0zTzOZTGLQoEFCq9WKnJwc83RU2Y0/fvx4ERQUZHVcbtSoUcLLy8vmc6ja92t3VVXu7vTz8xN5eXnm6bNnzxYArHZBjR49Wmi1WlFcXGyeZusxJ02aJFxdXc3zlZSUCF9fXxEdHW3R3po1awQAi93XqampwsHBweqYXuUxtT179pinJSUl1XjXdE1249t6LsnJyUKj0Vi8rjV9DXfv3i0AiLS0NIs2N2/ebDVdzTH7AwcOiMmTJwtvb28BQNx5551ixYoVNg9bVbcbPz8/X3h7e4sJEyZYTM/KyhJeXl4W0yuf90svvWQx75133imioqLM///zn/8Unp6e192lOn36dAHA4nXOz88XLVu2FC1atDDvpq/cZdiqVSur1+edd94RAMSvv/5qMV3NbvzK5fP9999b3TdixAgRGBhY7XOpzo8//igAiA8++MDqvqeffloAsHgv1UROTo7N10AIIVasWCEAiN9++01xX93c3MRjjz1mNf2bb74RAMTmzZsVtxkRESH69u1rNf1///ufACDeeustxW0OGjTI6jNZCCEKCwsFAPHcc88pbjMxMVE4OjravM/Pz0+MGjVKcZuLFy8WAMSZM2es7ouOjhZdunRR3GZdrKNV1cVufCGEWLt2rQAg9u/fX+O+KNqyv/fee7F3717cd999+OWXX7Bo0SLEx8ejWbNm+PLLLy3mvXY3hsFgwKVLl9CrVy/88ccfMBgMFvO2a9cOXbt2Nf8fExMDAOjbt6/F7rrK6bZ+JTt16lTz3xqNBlOnTkVpaSm2bdtm87kIIfD5559j8ODBEELg0qVL5lt8fDwMBoPqX86OGDECXl5eVv1++OGHLXZBxcTEoLS0FOfPnzdPu3a55efn49KlS+jRoweKiorw22+/AQAOHjyI3NxcTJgwwaK9MWPGoEmTJhZ9WbduHe644w60bdvW4jn27dsXwNVdbpXmzZsHIUStnbJ27XMpLCzEpUuX0K1bNwghbH5rvtFruG7dOnh5eeHee++1eC5RUVFwd3e3eC5qREdHIyUlBZmZmUhLS4OPjw+mTp2KoKAgPPzwwzhz5swN29i6dSvy8vIwevRoiz46OjoiJibGZh8nT55s8X+PHj0s1nFvb28UFhZe91fX3377LTp37ox77rnHPM3d3R0TJ07E6dOncezYMYv5ExISrHY15ubmAoDVOqRG5aEanU5ndZ9er6/2UM7NtHntPPZss7LmVm5Tq9XavK+uXvuG0mZ9qXyPXrp0qcY1TjeexVJ0dDTWr1+P0tJS/PLLL9iwYQNef/11DB8+HIcPH0a7du0AAHv27EFSUhL27t1rdQzYYDBYhOG1gQ7AfF9oaKjN6VWPzzo4OKBVq1YW09q0aQMA1Z4alZOTg7y8PKxcudJil+m1Ll68aHP6jdzM8/nf//6HOXPm4LvvvrPaRVP5JenPP/8EALRu3drificnJ7Ro0cJi2smTJ3H8+HH4+fnZ7Kva51gTZ86cwdy5c/Hll19avWZVv/DV5DU8efIkDAaD1e9DKl3vuWRlZVn87+XlVe1xNb1ej4ceeggjR45ESkoKZs2ahbS0NAwfPtzqta3q5MmTAGD+MlVV1V2wer3e6rVp0qSJxfKaMmUKPv30UwwYMADNmjVDXFwcRo4cif79+5vn+fPPP81fKq9VuXv2zz//tDhft2XLltU+B1GD37rcSOWytXVsuri4WNUxzRu1ee089myzsuZWbrO0tNTmfXX12jeUNutL5Xu0utP6bFEc9pW0Wq35Ry1t2rTBuHHjsG7dOiQlJSE9PR39+vVD27ZtsWTJEoSGhkKr1eLbb7/F66+/bnXM3dHR0eZjVDe9Nj6MKvvw8MMPIyEhweY8HTp0UNW22ueTl5eHXr16wdPTEy+99BLCw8Oh1+vx008/4dlnn7VabjVhMpkQGRmJJUuW2Ly/6heQ2lJRUYF7770Xly9fxrPPPou2bdvCzc0N58+fx9ixY1U/F39/f6Slpdm8v7ovNMDVY/PXWr16NcaOHWtz3uPHj2P16tVITU1FVlYWIiIiMH78ePTp06dGfQSuHrcPDAy0uv/aPTFA9evEtfz9/XH48GFs2bIFmzZtwqZNm7B69Wo8+uijeP/9929Yb4utDzJfX18AV798hoSEqGq3UuXyzszMtLovMzMTwcHBtd6mj4+Pza2066msqa5NAKr7WhdtXrsXsLba3LFjB4QQFsFxs21WVFTg4sWLFl/MS0tLkZube9OvfdXPrMzMTKtrPyhtsyq162h9qdwYaNq0aY1rVIf9tSp/4FC50L766iuUlJTgyy+/tNgSutndrNUxmUz4448/zFuCAMw/gKm6pVvJz88PHh4eqKioqJVfH9eGnTt3Ijc3F+vXr0fPnj3N0zMyMizmCwsLA3D1h43XBlB5eTlOnz5t8SUlPDwcv/zyC/r166foW+DN+vXXX/H777/j/ffft/hBXnW7omvyGoaHh2Pbtm3o3r274m/dVR83IiLC4n+DwYBPPvkE7733Hvbv3w93d3c8+OCDePzxx9GlSxer9qpbluHh4QCuBnRtrldarRaDBw/G4MGDYTKZMGXKFLz99tt48cUX0bp1a4SFheHEiRNWdZWHfirXmeupvIBHRkYGIiMjb6q/7du3h5OTEw4ePIiRI0eap5eWluLw4cMW02qqWbNm8PPzw8GDB63uO3DgADp16qS4TQcHB0RGRtpsc//+/WjVqhU8PDwUt9upUyfs3r0bJpPJ4kd6+/fvh6urq8V6rqTNHTt2wGg0Wuwh2r9/v/l+NW2+8847OH78uHmvbG20CVw93Dhw4EDz9IMHD8JkMt10m9cG+4ULF3Du3DmLMxRqqi7W0fqSkZEBBwcHReuRomP2ld8Aq/r2228BALfffjuAv7dWrp3XYDBg9erVSh5OkeXLl5v/FkJg+fLlcHZ2Rr9+/WzO7+joiGHDhuHzzz+3eXbBzV6RSw1by620tBRvvvmmxXx33303fH19sWrVKourR6WlpVntLh85ciTOnz+PVatWWT3elStXUFhYaP6/Nk+9s/VchBBWp4td60av4ciRI1FRUYEFCxZY1ZaXlyMvL6/atmNjYy1uld/q8/Pz8fDDDyMoKAiTJk2CRqPBO++8g8zMTLzzzjs2gx4A3NzcAMDqMePj4+Hp6Yl//etfNs8OULNeVR5Lr+Tg4GD+Qle5C3LgwIE4cOCAxalShYWFWLlyJVq0aGHxQV6dqKgoaLVam8GnlJeXF2JjY/Hhhx9a/Jo9NTUVBQUF5lMXlRo2bBi+/vpri9PMtm/fjt9//111m8OHD8ePP/5o8bxPnDiB77777qbazM7Oxvr1683TLl26hHXr1mHw4MGK90BUtllRUWFx2LGkpASrV69GTEyMqr10Q4YMgbOzs8VnjBACb731Fpo1a4Zu3bopbrNv377w8fGxOtU6JSUFrq6uGDRokOI2IyIi0LZtW6xcudLitNSUlBRoNBoMHz5ccZt1tY7Wh0OHDiEiIsLicPiNKNqynzZtGoqKinD//fejbdu2KC0txQ8//IBPPvkELVq0wLhx4wAAcXFx5i2RSZMmoaCgAKtWrYK/v7/NXSY3S6/XY/PmzUhISEBMTAw2bdqEb775Bs8///x1d+2++uqr2LFjB2JiYjBhwgS0a9cOly9fxk8//YRt27ZZnQNf17p164YmTZogISEBTz75JDQaDVJTU62+YGm1WsybNw/Tpk1D3759MXLkSJw+fRpr1qxBeHi4xVbnI488gk8//RSTJ0/Gjh070L17d1RUVOC3337Dp59+ii1btpj3zCxfvhzz58/Hjh07avQjvVOnTuHll1+2mn7nnXciLi4O4eHhmDVrFs6fPw9PT098/vnn1Z4PX5PXsFevXpg0aRKSk5Nx+PBhxMXFwdnZGSdPnsS6devwxhtvKH7T5+bmYsuWLZg8eTLGjx9vtcVfnU6dOsHR0RELFy6EwWCATqczX1ciJSUFjzzyCO666y6MGjUKfn5+OHPmDL755ht0797d4ktNTTz++OO4fPky+vbti5CQEPz5559YtmwZOnXqZD4m/9xzz+Gjjz7CgAED8OSTT8LHxwfvv/8+MjIy8Pnnn1udAmaLXq9HXFwctm3bhpdeeklRH2155ZVX0K1bN/Tq1QsTJ07EuXPn8O9//xtxcXEWvzcAru4p6dWrl8W1Cmx5/vnnsW7dOvTp0wf//Oc/UVBQgMWLFyMyMtL8+VOpco/QjS5pPGXKFKxatQqDBg3CrFmz4OzsjCVLliAgIABPPfWUxby9e/fGrl27bngocfjw4ejSpQvGjRuHY8eOoWnTpnjzzTdRUVGB+fPnW8w7duxY82tV3Z5I4OoPekeMGIHZs2fj4sWLaN26Nd5//32cPn0a7777rsW88+bNq9F7OSQkBNOnT8fixYtRVlaG6OhofPHFF9i9ezfS0tIsDjOtWbMG48aNu+4hMODqIaIFCxYgMTERI0aMQHx8PHbv3o0PP/wQr7zyCnx8fMzz7ty5E3369EFSUtINxy1ZvHgx7rvvPsTFxWHUqFE4evQoli9fjscff9zi1MHTp0+jZcuWSEhIwJo1a67bZl2so3/++SdSU1MBwPwFsvJzMiwsDI888sh162+krKwMu3btsrqOwQ3V+Hf7QohNmzaJxx57TLRt21a4u7ubL507bdo0qyvoffnll6JDhw5Cr9eLFi1aiIULF4r33ntPABAZGRnm+ao7hQuAxVWdhPj7FKBrryBVefW19PR0ERcXJ1xdXUVAQIBISkoyn250bZtVr6CXnZ0tEhMTRWhoqHB2dhaBgYGiX79+YuXKlTdcHtWdendt/4So/tSK1atXCwDixx9/NE/bs2eP6NKli3BxcRHBwcHm0xth4zSv//znPyIsLEzodDrRuXNnsWfPHhEVFWV1qcfS0lKxcOFCERERIXQ6nWjSpImIiooS8+fPt7hylNJT7wDYvI0fP14IIcSxY8dEbGyscHd3F02bNhUTJkwQv/zyi9VpXEpeQyGEWLlypYiKihIuLi7Cw8NDREZGimeeeUZcuHDBPE9NT70rLS1VfZnVVatWiVatWglHR0er5bZjxw4RHx8vvLy8hF6vF+Hh4WLs2LHi4MGDVs+7qsrXodJnn30m4uLihL+/v9BqtaJ58+Zi0qRJIjMz06IuPT1dDB8+XHh7ewu9Xi86d+4svv76a4t5rneajxBCrF+/Xmg0GotTnNReQU+Iq6dLduvWTej1euHn5ycSExOF0Wi0mCc/P18AqPEpWUePHjWvJ97e3mLMmDEiKyvLar6mTZvW+JSss2fPiuHDhwtPT0/h7u4u/vGPf4iTJ09azRcVFVXjU7IuX74sxo8fL3x9fYWrq6vo1auXxXu90rBhw4SLi4vNUzyrunLlipg1a5YIDAwUOp1OREdH2zyN76mnnqrxleUqKirEv/71LxEWFia0Wq2IiIgQH374odV8y5YtU3Ta4MqVK8Xtt98utFqtCA8PF6+//rowmUwW83z11VeKThvcsGGD6NSpk9DpdCIkJETMmTPH6tTvX3/9VdFpg7W9jla+x2zdbH0mKT31btOmTQKAzfXzehSFfUNUF5dabawqKiqEj4+PePzxx+3dFWqkysvLRZs2bcScOXPM0yrDftmyZSInJ8fiy1FCQoIIDQ0VOTk5NQorW7755huh0WjEkSNHbrb7ZpXnnlf9snMzjEajcHJyEsuXL6+1NoUQwt/fX8yaNatW24yOjhbDhw+v1TZHjBghoqOja7XNp59+WoSEhCi+PsL1rFixQri5udn8AqhWXayj5eXlIicnR3zxxRdWYW80GkVOTo7o1q2bVdgPGTJEDB06VPHjMewbqStXrlh9S67cU2DrWzlRTX388ceiSZMmIj8/Xwjxd9hX3q79UKq8MBAAmxf/qIlZs2apvl58dZYvX676evHV+frrr0VYWJjqPUG2HD16VHh4eFhc/OtmGQwGodVqxbFjx2qtTZPJJPz8/FRfL746d999t+oxDaozfPhw1WMaVKcu1tGff/652vfVkCFDbL6vjh07JhwdHa0ufFUTGiFq4Tw2Oxo7diw+++yzmx42s7HZuXMnZsyYgREjRsDX1xc//fSTeUjJQ4cOVXtRCyKliouL8d///tf8f4cOHcynVB07dgwXLlwAcPUiPtX9oJGILBUUFGDfvn3m/699Xx05csR83ZDael8x7Bup06dP48knn8SBAwdw+fJl+Pj4YODAgXj11VervegMERHdmhp92BMREdH1KTrPnoiIiBofhj0REZHkauVyuTIxmUy4cOECPDw86vXyskREVDuEEMjPz0dwcHCNLih1K2DYV3HhwoU6GxyGiIjqz9mzZ296UCdZMOyrUDPoBcmtJiPTVaX2d69qRgNUswdKTY2avlUd5a+mrh3zoS6pWQ4N/TfNaq67b2uYVxnw8/xvUu7fWLFiBVq0aAG9Xo+YmBgcOHCgxrXcdU9VaTSaers15P7VZ9/qS0NedvX5nGQl83NTSrqw/+STTzBz5kwkJSXhp59+QseOHREfH2++QAEREdGtRrqwX7JkCSZMmIBx48ahXbt2eOutt+Dq6or33nvP3l0jIiKyC6nCvrS0FIcOHUJsbKx5moODA2JjYy3G+b5WSUkJjEajxY2IiEgmUoX9pUuXUFFRgYCAAIvpAQEByMrKslmTnJwMLy8v842/xCciItlIFfZqzJ49GwaDwXw7e/asvbtERERUq6Q69a5p06ZwdHREdna2xfTs7GwEBgbarNHpdKpOVSEiImospNqy12q1iIqKwvbt283TTCYTtm/fjq5du9qxZ0RERPYj1ZY9AMycORMJCQm4++670blzZyxduhSFhYUYN26cvbtGRERkF9KF/YMPPoicnBzMnTsXWVlZ6NSpEzZv3mz1oz0iIqJbBcezr8JoNMLLy8ve3aAaUHMZWzWDYlRUVCiuUXNp2fok42Vi1VBzOV81v/FRe/lfWS9jW18MBgM8PT3t3Y0GQapj9kRERGSNYU9ERCQ5hj0REZHkGPZERESSY9gTERFJjmFPREQkOYY9ERGR5Bj2REREkmPYExERSY5hT0REJDmGPRERkeQY9kRERJKTbtQ7unWoGZhFzeA5agbCqU/1NaiNmmXn7OysuAZQ95zUDDZTVlZWL4+jlpplrmZwHw64Iz9u2RMREUmOYU9ERCQ5hj0REZHkGPZERESSY9gTERFJjmFPREQkOYY9ERGR5Bj2REREkmPYExERSY5hT0REJDmGPRERkeQY9kRERJJj2BMREUmOo95Ro2UymRTXFBcX10FPao9Wq1Vco2YEOzWjvTk4KN82aOjLu76oWXZq6xr6KI1kH9yyJyIikhzDnoiISHIMeyIiIskx7ImIiCTHsCciIpIcw56IiEhyDHsiIiLJMeyJiIgkx7AnIiKSHMOeiIhIcgx7IiIiyTHsiYiIJMeBcKjRUjNojEajUVyjZmARtYORlJeXK65RMyCQGmoGz3Fzc6u3xyotLVX1WEo5OjoqrnF2dlb1WGqeU32tD9S4cMueiIhIcgx7IiIiyTHsiYiIJMewJyIikhzDnoiISHIMeyIiIskx7ImIiCTHsCciIpIcw56IiEhyDHsiIiLJMeyJiIgkx7AnIiKSHAfCoUarIQ8ao5YQol4ex9XVVXGNmmVXWFiouKY+OTgo395RsxyKi4sV16ilZqAetQM3UePBLXsiIiLJMeyJiIgkJ13Yz5s3DxqNxuLWtm1be3eLiIjIbqQ8Zh8REYFt27aZ/3dykvJpEhER1YiUKejk5ITAwEB7d4OIiKhBkG43PgCcPHkSwcHBaNWqFcaMGYMzZ85UO29JSQmMRqPFjYiISCbShX1MTAzWrFmDzZs3IyUlBRkZGejRowfy8/Ntzp+cnAwvLy/zLTQ0tJ57TEREVLc0or5O7LWTvLw8hIWFYcmSJRg/frzV/SUlJSgpKTH/bzQaGfiNRH2dIy2j+jrPvj7PL1dDzTqk5iOzPj9meZ793wwGAzw9Pe3djQZBymP21/L29kabNm1w6tQpm/frdDrodLp67hUREVH9kW43flUFBQVIT09HUFCQvbtCRERkF9KF/axZs7Br1y6cPn0aP/zwA+6//344Ojpi9OjR9u4aERGRXUi3G//cuXMYPXo0cnNz4efnh3vuuQf79u2Dn5+fvbtGRERkF9L/QE8po9EILy8ve3eDakDNj8yKiooU16j5wZObm5viGgD1duqns7Oz4pqysjLFNWrfS2qWg5qPMjW/19FoNIpr6vOHivzh6t/4A72/Sbcbn4iIiCwx7ImIiCTHsCciIpIcw56IiEhyDHsiIiLJMeyJiIgkx7AnIiKSHMOeiIhIcgx7IiIiyTHsiYiIJMewJyIikhzDnoiISHLSjXpHtw41g9qo0aNHD8U1AwcOVPVYQUFBimvuuusuxTXt2rVTXFNRUaG4Rq3ly5crrlm6dKnimtOnTyuuUUPNYEqAugFqZB3Uhm4Ot+yJiIgkx7AnIiKSHMOeiIhIcgx7IiIiyTHsiYiIJMewJyIikhzDnoiISHIMeyIiIskx7ImIiCTHsCciIpIcw56IiEhyDHsiIiLJMeyJiIgkpxFCCHt3oiExGo3w8vJSXKfRaBTXqF309flYSjk7OyuuUdu38vJyxTUvvPCC4pqHHnpIcY2aUeUAoLi4WHHNm2++qbhGzchoaka969ixo+IaAOjfv7/iml9++UVxzciRIxXX/P7774pr1Lxngfp738rKYDDA09PT3t1oELhlT0REJDmGPRERkeQY9kRERJJj2BMREUmOYU9ERCQ5hj0REZHkGPZERESSY9gTERFJjmFPREQkOYY9ERGR5Bj2REREkmPYExERSY4D4VShdiAcJycnxTVqBnJRS6vVKq5Rs2qUlZUprlHr/vvvV1yTlpamuGbr1q2Ka5YvX664Ru1jubu7K65Rs77m5eUprlGrT58+imvmzZunuKakpERxzfDhwxXXlJaWKq4B1L2f1AxYJCsOhPM3btkTERFJjmFPREQkOYY9ERGR5Bj2REREkmPYExERSY5hT0REJDmGPRERkeQY9kRERJJj2BMREUmOYU9ERCQ5hj0REZHkGPZERESS40A4VdTnQDiOjo6KawB1g3c4OzsrrlHTv+LiYsU1fn5+imsA4LvvvlNck5mZqbhmypQpimtycnIU1wBXB+5QSs1rq2aAFTUD7hQUFCiuAdSte506dVJc8+OPPyqu8fb2VlxjNBoV1wCAh4eH4pr8/HxVjyUjDoTzN27ZExERSY5hT0REJLlGFfbff/89Bg8ejODgYGg0GnzxxRcW9wshMHfuXAQFBcHFxQWxsbE4efKkfTpLRETUQDSqsC8sLETHjh2xYsUKm/cvWrQI//nPf/DWW29h//79cHNzQ3x8vKrjyERERLJQ/qsyOxowYAAGDBhg8z4hBJYuXYo5c+ZgyJAhAIAPPvgAAQEB+OKLLzBq1Kj67CoREVGD0ai27K8nIyMDWVlZiI2NNU/z8vJCTEwM9u7dW21dSUkJjEajxY2IiEgm0oR9VlYWACAgIMBiekBAgPk+W5KTk+Hl5WW+hYaG1mk/iYiI6ps0Ya/W7NmzYTAYzLezZ8/au0tERES1SpqwDwwMBABkZ2dbTM/OzjbfZ4tOp4Onp6fFjYiISCbShH3Lli0RGBiI7du3m6cZjUbs378fXbt2tWPPiIiI7KtR/Rq/oKAAp06dMv+fkZGBw4cPw8fHB82bN8f06dPx8ssv47bbbkPLli3x4osvIjg4GEOHDrVfp4mIiOysUYX9wYMH0adPH/P/M2fOBAAkJCRgzZo1eOaZZ1BYWIiJEyciLy8P99xzDzZv3gy9Xm+vLhMREdkdB8KpQu1AOGqoGeQCkG+gizFjxqiq+/DDDxXX3HvvvYprtm3bprhGLV9fX8U1ataH0tJSxTVqfs+idl11dXVVXFNYWKi4ZsuWLYprxo0bp7jmwoULimsAQKvVKq5R89rKigPh/E2aY/ZERERkG8OeiIhIcgx7IiIiyTHsiYiIJMewJyIikhzDnoiISHIMeyIiIskx7ImIiCTHsCciIpIcw56IiEhyDHsiIiLJMeyJiIgkx7AnIiKSXKMa4lY2ZWVl9u5CrdPpdIprmjdvruqxzpw5o7hm3759imuCgoIU1+Tk5CiuAYDc3FxVdUo5OSl/66sZVU7N6HX1+VgJCQmKa7KyshTXqFneAEewo9rDLXsiIiLJMeyJiIgkx7AnIiKSHMOeiIhIcgx7IiIiyTHsiYiIJMewJyIikhzDnoiISHIMeyIiIskx7ImIiCTHsCciIpIcw56IiEhyHAinGjqdDhqNpsbzFxcXK34MNTVq6fV6xTVq+qdm4I5jx44prgGAgIAAxTVCCMU1mZmZimtcXFwU1wBAeXm54ho1gw+peZ3ULDs1A9qo5ebmprjmjjvuUFyjZiAcNa8RoG59cHBQvg1nMpkU11Djwi17IiIiyTHsiYiIJMewJyIikhzDnoiISHIMeyIiIskx7ImIiCTHsCciIpIcw56IiEhyDHsiIiLJMeyJiIgkx7AnIiKSHMOeiIhIchwIpxrl5eWKBsJp6NQMYlJfj7Nnzx5Vj6VmgJrnnntOcU1qaqrimt9//11xjVpq1lNHR0fFNWoGZVFr0KBBimu6d++uuGb27NmKa1q3bq245vTp04pr1OKgNmQLt+yJiIgkx7AnIiKSHMOeiIhIcgx7IiIiyTHsiYiIJMewJyIikhzDnoiISHIMeyIiIskx7ImIiCTHsCciIpIcw56IiEhyDHsiIiLJaUR9jZDSSBiNRnh5eSmu0+v1imvKysoU1wBARUWFqrr6oGaAFbXPZ8yYMYprPvzwQ8U1586dU1yzZMkSxTUAcOnSJcU1x48fV1wTGRmpuCYwMFBxzcCBAxXXAEBwcLDimlatWimuKSkpUVzTrFkzxTW5ubmKawDAwUH59hgHwvmbwWCAp6envbvRIHDLnoiISHIMeyIiIsk1qrD//vvvMXjwYAQHB0Oj0eCLL76wuH/s2LHQaDQWt/79+9uns0RERA1Eowr7wsJCdOzYEStWrKh2nv79+yMzM9N8++ijj+qxh0RERA2Pk707oMSAAQMwYMCA686j0+lU/ZCIiIhIVo1qy74mdu7cCX9/f9x+++144oknbvgr2JKSEhiNRosbERGRTKQK+/79++ODDz7A9u3bsXDhQuzatQsDBgy47qldycnJ8PLyMt9CQ0PrscdERER1r1Htxr+RUaNGmf+OjIxEhw4dEB4ejp07d6Jfv342a2bPno2ZM2ea/zcajQx8IiKSilRb9lW1atUKTZs2xalTp6qdR6fTwdPT0+JGREQkE6nD/ty5c8jNzUVQUJC9u0JERGQ3jWo3fkFBgcVWekZGBg4fPgwfHx/4+Phg/vz5GDZsGAIDA5Geno5nnnkGrVu3Rnx8vB17TUREZF+NKuwPHjyIPn36mP+vPNaekJCAlJQUHDlyBO+//z7y8vIQHByMuLg4LFiwADqdzl5dJiIisjsOhFOF2oFwqHF44YUXFNc8+eSTimvc3d0V1wCAq6ur4prS0lLFNRqNRnGNs7Oz4hq1DAaD4ho1g9r4+/srrvH19VVcU1BQoLgGAMrLyxXXcCCcv3EgnL9JfcyeiIiIGPZERETSY9gTERFJjmFPREQkOYY9ERGR5Bj2REREkmPYExERSY5hT0REJDmGPRERkeQY9kRERJJj2BMREUmOYU9ERCQ5hj0REZHkOOpdFRz1rvFQM3SxmpHRgoODFdf07NlTcQ0AREZGKq7Zt2+f4hpHR0fFNWqe0+bNmxXXAEBxcbHimgEDBiiuSUxMVFzTqlUrxTWXLl1SXKOWm5ub4prCwsI66In9cdS7v3HLnoiISHIMeyIiIskx7ImIiCTHsCciIpIcw56IiEhyDHsiIiLJMeyJiIgkx7AnIiKSHMOeiIhIcgx7IiIiyTHsiYiIJMewJyIikhwHwqmCA+E0HhqNRnGNmgFgysvLFdeo1ZAHMdHr9Ypr1AxoA6h7bZ966inFNY899pjiGjWDFVVUVCiuoZvHgXD+xi17IiIiyTHsiYiIJMewJyIikhzDnoiISHIMeyIiIskx7ImIiCTHsCciIpIcw56IiEhyDHsiIiLJMeyJiIgkx7AnIiKSHMOeiIhIck727gCRWmrGcHJyUr7KqxkApqCgQHENAFy5ckVVXX0oKSlRXKNm4CFA3cAx7u7uimvS09MV16jpm1arVVwDqBsQSM3rRPLjlj0REZHkGPZERESSY9gTERFJjmFPREQkOYY9ERGR5Bj2REREkmPYExERSY5hT0REJDmGPRERkeQY9kRERJJj2BMREUmOYU9ERCQ5DoRDjZaaQVaKi4vroCfW1AxgAqh7TmoGBFJT4+zsrLimvLxccY1azZo1U1xTVFSkuEbNgDtqB0ZSQ6fTKa7h4Dny45Y9ERGR5Bj2REREkmtUYZ+cnIzo6Gh4eHjA398fQ4cOxYkTJyzmKS4uRmJiInx9feHu7o5hw4YhOzvbTj0mIiKyv0YV9rt27UJiYiL27duHrVu3oqysDHFxcSgsLDTPM2PGDHz11VdYt24ddu3ahQsXLuCBBx6wY6+JiIjsq1H9QG/z5s0W/69Zswb+/v44dOgQevbsCYPBgHfffRdr165F3759AQCrV6/GHXfcgX379qFLly726DYREZFdNaot+6oMBgMAwMfHBwBw6NAhlJWVITY21jxP27Zt0bx5c+zdu9dmGyUlJTAajRY3IiIimTTasDeZTJg+fTq6d++O9u3bAwCysrKg1Wrh7e1tMW9AQACysrJstpOcnAwvLy/zLTQ0tK67TkREVK8abdgnJibi6NGj+Pjjj2+qndmzZ8NgMJhvZ8+eraUeEhERNQyN6ph9palTp+Lrr7/G999/j5CQEPP0wMBAlJaWIi8vz2LrPjs7G4GBgTbb0ul0qi5CQURE1Fg0qi17IQSmTp2KDRs24LvvvkPLli0t7o+KioKzszO2b99unnbixAmcOXMGXbt2re/uEhERNQiNass+MTERa9euxcaNG+Hh4WE+Du/l5QUXFxd4eXlh/PjxmDlzJnx8fODp6Ylp06aha9eu/CU+ERHdshpV2KekpAAAevfubTF99erVGDt2LADg9ddfh4ODA4YNG4aSkhLEx8fjzTffrOeeEhERNRwaoWZEDIkZjUZ4eXnZuxtUA1qtVnGNq6ur4pprL9pUU2VlZYpr1FLznNQMCGQymRTXqKXmrJgtW7YorklLS1Ncs3jxYsU1paWlimvUUjNgUX2ur/XJYDDA09PT3t1oEBrVMXsiIiJSjmFPREQkOYY9ERGR5Bj2REREkmPYExERSY5hT0REJDmGPRERkeQY9kRERJJj2BMREUmOYU9ERCQ5hj0REZHkGPZERESSY9gTERFJrlENcUt0LTUjidXX6GNqR9oyGo2Ka4qKilQ9llJ6vV5xjZrR9QAgICBAcY23t7fimo8//lhxjZp1yMFB3XaVk5Pyj+jy8nJVj0Vy45Y9ERGR5Bj2REREkmPYExERSY5hT0REJDmGPRERkeQY9kRERJJj2BMREUmOYU9ERCQ5hj0REZHkGPZERESSY9gTERFJjmFPREQkOQ6EQ42WmoFZ1AwSoqamvgbcAQAXFxfFNWoGqFFT4+HhobhGbZ2a1ykjI0NxjRoajUZVXX2uRyQ3btkTERFJjmFPREQkOYY9ERGR5Bj2REREkmPYExERSY5hT0REJDmGPRERkeQY9kRERJJj2BMREUmOYU9ERCQ5hj0REZHkGPZERESS40A41GipGZilvtRn365cuVJvj6VUYWGhqjo1A8C4uroqrnF0dFRcYzKZ6qVGLXd3d8U1BQUFddATaki4ZU9ERCQ5hj0REZHkGPZERESSY9gTERFJjmFPREQkOYY9ERGR5Bj2REREkmPYExERSY5hT0REJDmGPRERkeQY9kRERJJj2BMREUmOA+EQUZ3RaDSq6rRareIaNQMClZWVKa5xdnZWXCOEUFwDAE5Oyj+iOagN2cIteyIiIskx7ImIiCTXqMI+OTkZ0dHR8PDwgL+/P4YOHYoTJ05YzNO7d29oNBqL2+TJk+3UYyIiIvtrVGG/a9cuJCYmYt++fdi6dSvKysoQFxeHwsJCi/kmTJiAzMxM823RokV26jEREZH9Naof6G3evNni/zVr1sDf3x+HDh1Cz549zdNdXV0RGBhY390jIiJqkBrVln1VBoMBAODj42MxPS0tDU2bNkX79u0xe/ZsFBUVVdtGSUkJjEajxY2IiEgmjWrL/lomkwnTp09H9+7d0b59e/P0hx56CGFhYQgODsaRI0fw7LPP4sSJE1i/fr3NdpKTkzF//vz66jYREVG90wi1J4Da2RNPPIFNmzbhv//9L0JCQqqd77vvvkO/fv1w6tQphIeHW91fUlKCkpIS8/9GoxGhoaF10meiW42jo6OqumsPy9XUBx98oLhGzXu9oZ9nX1xcrOqxZGQwGODp6WnvbjQIjXLLfurUqfj666/x/fffXzfoASAmJgYAqg17nU4HnU5XJ/0kIiJqCBpV2AshMG3aNGzYsAE7d+5Ey5Ytb1hz+PBhAEBQUFAd946IiKhhalRhn5iYiLVr12Ljxo3w8PBAVlYWAMDLywsuLi5IT0/H2rVrMXDgQPj6+uLIkSOYMWMGevbsiQ4dOti590RERPbRqMI+JSUFwNUL51xr9erVGDt2LLRaLbZt24alS5eisLAQoaGhGDZsGObMmWOH3hIRETUMjSrsb/Qjl9DQUOzataueekNERNQ4NKqwJ6LGpaKiQlVd1Wtn1ISaUe8CAgIU12RnZyuuUUvN6H/e3t6Ka/Ly8hTXUOPSqC+qQ0RERDfGsCciIpIcw56IiEhyDHsiIiLJMeyJiIgkx7AnIiKSHMOeiIhIcgx7IiIiyTHsiYiIJMewJyIikhzDnoiISHIMeyIiIslpxI2GkrvFGI1GeHl52bsbRFLQaDSq6tR8LDXkAWB8fX1V1eXm5tZyT24tBoMBnp6e9u5Gg8AteyIiIskx7ImIiCTHsCciIpIcw56IiEhyDHsiIiLJMeyJiIgkx7AnIiKSHMOeiIhIcgx7IiIiyTHsiYiIJMewJyIikpyTvTvQ0HCoAKLaU5/vp4b83jWZTPbuwi2pIa8T9Y1hX0V+fr69u0BEKhgMBnt3oVp//fWXvbtwS8rPz+fAZv8fR72rwmQy4cKFC/Dw8LAasctoNCI0NBRnz569pUdS4nK4isvhKi6Hq7gcrmoIy0EIgfz8fAQHB8PBgUerAW7ZW3FwcEBISMh15/H09Lyl38yVuByu4nK4isvhKi6Hq+y9HLhFb4lfeYiIiCTHsCciIpIcw14BnU6HpKQk6HQ6e3fFrrgcruJyuIrL4Souh6u4HBom/kCPiIhIctyyJyIikhzDnoiISHIMeyIiIskx7ImIiCTHsCciIpIcw76GVqxYgRYtWkCv1yMmJgYHDhywd5fq1bx586DRaCxubdu2tXe36tz333+PwYMHIzg4GBqNBl988YXF/UIIzJ07F0FBQXBxcUFsbCxOnjxpn87WoRsth7Fjx1qtH/3797dPZ+tQcnIyoqOj4eHhAX9/fwwdOhQnTpywmKe4uBiJiYnw9fWFu7s7hg0bhuzsbDv1uG7UZDn07t3bap2YPHmynXpMDPsa+OSTTzBz5kwkJSXhp59+QseOHREfH4+LFy/au2v1KiIiApmZmebbf//7X3t3qc4VFhaiY8eOWLFihc37Fy1ahP/85z946623sH//fri5uSE+Ph7FxcX13NO6daPlAAD9+/e3WD8++uijeuxh/di1axcSExOxb98+bN26FWVlZYiLi0NhYaF5nhkzZuCrr77CunXrsGvXLly4cAEPPPCAHXtd+2qyHABgwoQJFuvEokWL7NRjgqAb6ty5s0hMTDT/X1FRIYKDg0VycrIde1W/kpKSRMeOHe3dDbsCIDZs2GD+32QyicDAQLF48WLztLy8PKHT6cRHH31khx7Wj6rLQQghEhISxJAhQ+zSH3u6ePGiACB27dolhLj6+js7O4t169aZ5zl+/LgAIPbu3Wuvbta5qstBCCF69eol/vnPf9qvU2SBW/Y3UFpaikOHDiE2NtY8zcHBAbGxsdi7d68de1b/Tp48ieDgYLRq1QpjxozBmTNn7N0lu8rIyEBWVpbFuuHl5YWYmJhbbt0AgJ07d8Lf3x+33347nnjiCeTm5tq7S3WuclhdHx8fAMChQ4dQVlZmsU60bdsWzZs3l3qdqLocKqWlpaFp06Zo3749Zs+ejaKiInt0j8BR727o0qVLqKioQEBAgMX0gIAA/Pbbb3bqVf2LiYnBmjVrcPvttyMzMxPz589Hjx49cPToUXh4eNi7e3aRlZUFADbXjcr7bhX9+/fHAw88gJYtWyI9PR3PP/88BgwYgL1798LR0dHe3asTJpMJ06dPR/fu3dG+fXsAV9cJrVYLb29vi3llXidsLQcAeOihhxAWFobg4GAcOXIEzz77LE6cOIH169fbsbe3LoY91ciAAQPMf3fo0AExMTEICwvDp59+ivHjx9uxZ9QQjBo1yvx3ZGQkOnTogPDwcOzcuRP9+vWzY8/qTmJiIo4ePXpL/HbleqpbDhMnTjT/HRkZiaCgIPTr1w/p6ekIDw+v727e8rgb/waaNm0KR0dHq1/TZmdnIzAw0E69sj9vb2+0adMGp06dsndX7Kby9ee6Ya1Vq1Zo2rSptOvH1KlT8fXXX2PHjh0ICQkxTw8MDERpaSny8vIs5pd1nahuOdgSExMDANKuEw0dw/4GtFotoqKisH37dvM0k8mE7du3o2vXrnbsmX0VFBQgPT0dQUFB9u6K3bRs2RKBgYEW64bRaMT+/ftv6XUDAM6dO4fc3Fzp1g8hBKZOnYoNGzbgu+++Q8uWLS3uj4qKgrOzs8U6ceLECZw5c0aqdeJGy8GWw4cPA4B060Rjwd34NTBz5kwkJCTg7rvvRufOnbF06VIUFhZi3Lhx9u5avZk1axYGDx6MsLAwXLhwAUlJSXB0dMTo0aPt3bU6VVBQYLElkpGRgcOHD8PHxwfNmzfH9OnT8fLLL+O2225Dy5Yt8eKLLyI4OBhDhw61X6frwPWWg4+PD+bPn49hw4YhMDAQ6enpeOaZZ9C6dWvEx8fbsde1LzExEWvXrsXGjRvh4eFhPg7v5eUFFxcXeHl5Yfz48Zg5cyZ8fHzg6emJadOmoWvXrujSpYude197brQc0tPTsXbtWgwcOBC+vr44cuQIZsyYgZ49e6JDhw527v0tyt6nAzQWy5YtE82bNxdarVZ07txZ7Nu3z95dqlcPPvigCAoKElqtVjRr1kw8+OCD4tSpU/buVp3bsWOHAGB1S0hIEEJcPf3uxRdfFAEBAUKn04l+/fqJEydO2LfTdeB6y6GoqEjExcUJPz8/4ezsLMLCwsSECRNEVlaWvbtd62wtAwBi9erV5nmuXLkipkyZIpo0aSJcXV3F/fffLzIzM+3X6Tpwo+Vw5swZ0bNnT+Hj4yN0Op1o3bq1ePrpp4XBYLBvx29hHM+eiIhIcjxmT0REJDmGPRERkeQY9kRERJJj2BMREUmOYU9ERCQ5hj0REZHkGPZERESSY9gTERFJjmFPREQkOYY9ERGR5Bj2REREkvt/0dVo9j4vo6cAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_features, train_labels = next(iter(train_dataloader))\n",
    "img = train_features[0].squeeze()\n",
    "label = train_labels[0]\n",
    "plt.title(f\"Sample Image: Label->{label}\")\n",
    "plt.imshow(img.reshape(28,28), cmap=\"gray\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ad678500",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature batch shape: torch.Size([64, 784])\n",
      "Labels batch shape: torch.Size([64, 1, 10])\n"
     ]
    }
   ],
   "source": [
    "print(f\"Feature batch shape: {train_features.size()}\")\n",
    "print(f\"Labels batch shape: {train_labels.size()}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8fed589d",
   "metadata": {},
   "source": [
    "#### Simple Neural Network for Predicting Class of HandWritten Digits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8361b5dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch import nn\n",
    "from torch.nn import functional as F\n",
    "\n",
    "class NeuralNet(nn.Module):\n",
    "    def __init__(self, h1, h2):\n",
    "        super(NeuralNet, self).__init__() #Initializing superclass\n",
    "        self.layer1 = nn.Linear(784, h1) #784 is the number of input features\n",
    "        self.layer2 = nn.Linear(h1, h2)\n",
    "        self.layer3 = nn.Linear(h2, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.layer1(x)\n",
    "        x = self.layer2(x)\n",
    "        x = self.layer3(x)\n",
    "        out = F.log_softmax(x, dim=1)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14225387",
   "metadata": {},
   "source": [
    "### Training Loop in PyTorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f5e6cdc2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0: 1.8438410758972168\n",
      "1: 0.9629454612731934\n",
      "2: 0.6244114637374878\n",
      "3: 0.5058785676956177\n",
      "4: 0.44916823506355286\n",
      "5: 0.41618776321411133\n",
      "6: 0.3942669928073883\n",
      "7: 0.37872496247291565\n",
      "8: 0.36644017696380615\n",
      "9: 0.3569311201572418\n",
      "10: 0.34884756803512573\n",
      "11: 0.3423047661781311\n",
      "12: 0.3365049362182617\n",
      "13: 0.3313397765159607\n",
      "14: 0.32687366008758545\n",
      "15: 0.3229764997959137\n",
      "16: 0.3197023868560791\n",
      "17: 0.3167540431022644\n",
      "18: 0.31363245844841003\n",
      "19: 0.3109292984008789\n",
      "20: 0.30855804681777954\n",
      "21: 0.30633190274238586\n",
      "22: 0.30406874418258667\n",
      "23: 0.3020666837692261\n",
      "24: 0.30048638582229614\n",
      "25: 0.2988789677619934\n",
      "26: 0.29702264070510864\n",
      "27: 0.2954942286014557\n",
      "28: 0.294420450925827\n",
      "29: 0.29282376170158386\n",
      "30: 0.29156607389450073\n",
      "31: 0.29025980830192566\n",
      "32: 0.2890019416809082\n",
      "33: 0.2877555191516876\n",
      "34: 0.28678199648857117\n",
      "35: 0.28576675057411194\n",
      "36: 0.2848820686340332\n",
      "37: 0.28378960490226746\n",
      "38: 0.2828204929828644\n",
      "39: 0.2818012237548828\n",
      "40: 0.2811521589756012\n",
      "41: 0.2804977297782898\n",
      "42: 0.2794405519962311\n",
      "43: 0.27882421016693115\n",
      "44: 0.27799978852272034\n",
      "45: 0.2773149907588959\n",
      "46: 0.27658605575561523\n",
      "47: 0.2759048640727997\n",
      "48: 0.27533483505249023\n",
      "49: 0.27449366450309753\n"
     ]
    }
   ],
   "source": [
    "# Constants\n",
    "lr = 0.005\n",
    "epochs = 50\n",
    "\n",
    "# Loading Dataset\n",
    "train_dataset = CustomMNISTDataset('../chapter3/output/training/')\n",
    "train_dataloader = DataLoader(train_dataset, batch_size=64, shuffle=True)\n",
    "\n",
    "# Initialize Model\n",
    "model = NeuralNet(512, 128)\n",
    "# Initialize Optimizer\n",
    "optimizer = optim.Adadelta(model.parameters(), lr=0.005)\n",
    "# Loss function\n",
    "loss_fn = nn.CrossEntropyLoss()\n",
    "\n",
    "# set model to train mode\n",
    "losses = []\n",
    "model.train()\n",
    "for epoch in range(epochs):\n",
    "\tbatch_losses = []    \n",
    "\tfor batch_idx, (data, targets) in enumerate(train_dataloader):\n",
    "\t\toptimizer.zero_grad()\n",
    "\t\toutputs = model(data)\n",
    "\t\tloss = loss_fn(outputs, targets.squeeze(1).float())\n",
    "\t\tloss.backward() # Backpropagation of loss\n",
    "\t\toptimizer.step() # Updating Weights based on Gradients\n",
    "\t\tbatch_losses.append(loss.item())\n",
    "\tfinal_epoch_loss = torch.mean(torch.Tensor(batch_losses))\n",
    "\tlosses.append(final_epoch_loss)\n",
    "\tprint(f\"{epoch}: {final_epoch_loss}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c4ac119",
   "metadata": {},
   "source": [
    "#### Plotting Training Loss and Epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "152696c0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHHCAYAAABDUnkqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAABNjUlEQVR4nO3deXwTZf4H8M8kaZKmNxRKC4VCEeRYSuWoFRGRChSogrgisFLAY1EuKR4gCsIqVVRwXRDWE3c9uBRQOQQ5F34gUKgicl8tpQfl6N2mTZ7fH2lSYgttIJNp08/79cornclM8s3Y3X54jnkkIYQAERERkZtQKV0AERERkTMx3BAREZFbYbghIiIit8JwQ0RERG6F4YaIiIjcCsMNERERuRWGGyIiInIrDDdERETkVhhuiIiIyK0w3BC5gdGjRyMsLOyWzn399dchSZJzCyK3EhYWhkGDBildBlGNMdwQyUiSpBo9tm/frnSpihg9ejS8vb2VLkNxYWFhN/zd6N+/v9LlEdU5GqULIHJn//3vf+22//Of/2Dz5s2V9rdr1+62Pufjjz+G2Wy+pXNfffVVTJs27bY+n25f586dMXXq1Er7Q0JCFKiGqG5juCGS0d/+9je77b1792Lz5s2V9v9ZYWEhDAZDjT/Hw8PjluoDAI1GA42G/1egtKZNm1b7e0FENcNuKSKF3X///ejYsSOSkpJw3333wWAw4JVXXgEArF27FgMHDkRISAh0Oh3Cw8Pxj3/8AyaTye49/jzm5ty5c5AkCe+++y4++ugjhIeHQ6fToVu3bti/f7/duVWNuZEkCRMmTMCaNWvQsWNH6HQ6dOjQARs3bqxU//bt29G1a1fo9XqEh4fj3//+t9PH8axcuRJdunSBp6cnAgMD8be//Q1paWl2x2RkZGDMmDFo1qwZdDodgoOD8fDDD+PcuXO2Yw4cOIB+/fohMDAQnp6eaNmyJcaOHXvTzx40aBBatWpV5WvR0dHo2rWrbXvz5s2499574e/vD29vb7Rt29b239IZrN14Z86cQb9+/eDl5YWQkBDMmTMHQgi7YwsKCjB16lSEhoZCp9Ohbdu2ePfddysdBwBffvklunfvDoPBgICAANx3333YtGlTpeN27dqF7t27Q6/Xo1WrVvjPf/7jtO9G5Ez85xpRLXD58mXExsbi8ccfx9/+9jcEBQUBAJYuXQpvb28kJCTA29sbW7duxcyZM5Gbm4t33nmn2vf9+uuvkZeXh7///e+QJAnz5s3DI488gjNnzlTb2rNr1y589913eO655+Dj44MPPvgAQ4cORUpKCho2bAgAOHToEPr374/g4GDMnj0bJpMJc+bMQaNGjW7/opRbunQpxowZg27duiExMRGZmZn45z//id27d+PQoUPw9/cHAAwdOhRHjhzBxIkTERYWhqysLGzevBkpKSm27b59+6JRo0aYNm0a/P39ce7cOXz33Xc3/fxhw4Zh1KhR2L9/P7p162bbf/78eezdu9f23+HIkSMYNGgQOnXqhDlz5kCn0+HUqVPYvXt3jb5naWkpsrOzK+338vKCp6enbdtkMqF///64++67MW/ePGzcuBGzZs1CWVkZ5syZAwAQQuChhx7Ctm3b8OSTT6Jz58746aef8OKLLyItLQ0LFiywvd/s2bPx+uuv45577sGcOXOg1Wrxyy+/YOvWrejbt6/tuFOnTuHRRx/Fk08+ifj4eHz22WcYPXo0unTpgg4dOtToOxK5jCAilxk/frz48//sevXqJQCIJUuWVDq+sLCw0r6///3vwmAwiOLiYtu++Ph40aJFC9v22bNnBQDRsGFDceXKFdv+tWvXCgDihx9+sO2bNWtWpZoACK1WK06dOmXb9+uvvwoA4l//+pdtX1xcnDAYDCItLc227+TJk0Kj0VR6z6rEx8cLLy+vG75uNBpF48aNRceOHUVRUZFt/48//igAiJkzZwohhLh69aoAIN55550bvtfq1asFALF///5q67peTk6O0Ol0YurUqXb7582bJyRJEufPnxdCCLFgwQIBQFy6dMmh9xdCiBYtWggAVT4SExNtx8XHxwsAYuLEibZ9ZrNZDBw4UGi1Wttnr1mzRgAQb7zxht3nPProo0KSJNt/15MnTwqVSiWGDBkiTCaT3bFms7lSfTt37rTty8rKqvK6ENUG7JYiqgV0Oh3GjBlTaf/1/2LPy8tDdnY2evbsicLCQhw7dqza9x02bBgCAgJs2z179gQAnDlzptpzY2JiEB4ebtvu1KkTfH19beeaTCb8/PPPGDx4sN2g19atWyM2Nrba96+JAwcOICsrC8899xz0er1t/8CBA3HnnXdi3bp1ACzXSavVYvv27bh69WqV72Vt4fnxxx9RWlpa4xp8fX0RGxuLFStW2HXpLF++HHfffTeaN29u9/5r1669pcHdUVFR2Lx5c6XH8OHDKx07YcIE28/WLkSj0Yiff/4ZALB+/Xqo1WpMmjTJ7rypU6dCCIENGzYAANasWQOz2YyZM2dCpbL/c/DnbsX27dvbfn8AoFGjRmjbtm2NfpeIXI3hhqgWaNq0KbRabaX9R44cwZAhQ+Dn5wdfX180atTINug0Jyen2ve1/uG1sgadGwWAm51rPd96blZWFoqKitC6detKx1W171acP38eANC2bdtKr915552213U6Hd5++21s2LABQUFBuO+++zBv3jxkZGTYju/VqxeGDh2K2bNnIzAwEA8//DA+//xzlJSUVFvHsGHDkJqaij179gAATp8+jaSkJAwbNszumB49euCpp55CUFAQHn/8caxYsaLGQScwMBAxMTGVHi1atLA7TqVSVRoD1KZNGwCwjS86f/48QkJC4OPjY3ecdVae9bqdPn0aKpUK7du3r7a+6n4fiGoThhuiWuD6Fhqra9euoVevXvj1118xZ84c/PDDD9i8eTPefvttAKjRH021Wl3lflHFoFJnnquE559/HidOnEBiYiL0ej1ee+01tGvXDocOHQJgaYlYtWoV9uzZgwkTJiAtLQ1jx45Fly5dkJ+ff9P3jouLg8FgwIoVKwAAK1asgEqlwl//+lfbMZ6enti5cyd+/vlnPPHEE/jtt98wbNgwPPjgg5UGgNdFde33geo3hhuiWmr79u24fPkyli5dismTJ2PQoEGIiYmx62ZSUuPGjaHX63Hq1KlKr1W171ZYWy2OHz9e6bXjx49XatUIDw/H1KlTsWnTJvz+++8wGo1477337I65++678eabb+LAgQP46quvcOTIESxbtuymdXh5eWHQoEFYuXIlzGYzli9fjp49e1a6B41KpUKfPn0wf/58/PHHH3jzzTexdetWbNu27Va+fpXMZnOlrqATJ04AgG3GXIsWLXDx4kXk5eXZHWftyrRet/DwcJjNZvzxxx9Oq4+oNmC4IaqlrP9Svv5fxkajER9++KFSJdlRq9WIiYnBmjVrcPHiRdv+U6dO2cZ03K6uXbuicePGWLJkiV330YYNG3D06FEMHDgQgOW+QMXFxXbnhoeHw8fHx3be1atXK7UydO7cGQBq3DV18eJFfPLJJ/j111/tuqQA4MqVK5XOceT9HbFw4ULbz0IILFy4EB4eHujTpw8AYMCAATCZTHbHAcCCBQsgSZJtTNTgwYOhUqkwZ86cSi2BbJGhuoxTwYlqqXvuuQcBAQGIj4/HpEmTIEkS/vvf/9aqPzqvv/46Nm3ahB49euDZZ5+1/UHt2LEjkpOTa/QepaWleOONNyrtb9CgAZ577jm8/fbbGDNmDHr16oXhw4fbpoKHhYVhypQpACwtF3369MFjjz2G9u3bQ6PRYPXq1cjMzMTjjz8OAPjiiy/w4YcfYsiQIQgPD0deXh4+/vhj+Pr6YsCAAdXWOWDAAPj4+OCFF16AWq3G0KFD7V6fM2cOdu7ciYEDB6JFixbIysrChx9+iGbNmuHee++t9v3T0tLw5ZdfVtrv7e2NwYMH27b1ej02btyI+Ph4REVFYcOGDVi3bh1eeeUV2xT8uLg49O7dGzNmzMC5c+cQERGBTZs2Ye3atXj++edtA8Vbt26NGTNm4B//+Ad69uyJRx55BDqdDvv370dISAgSExOrrZuoVlJqmhZRfXSjqeAdOnSo8vjdu3eLu+++W3h6eoqQkBDx0ksviZ9++kkAENu2bbMdd6Op4FVNjQYgZs2aZdu+0VTw8ePHVzq3RYsWIj4+3m7fli1bRGRkpNBqtSI8PFx88sknYurUqUKv19/gKlSwTm2u6hEeHm47bvny5SIyMlLodDrRoEEDMXLkSHHhwgXb69nZ2WL8+PHizjvvFF5eXsLPz09ERUWJFStW2I45ePCgGD58uGjevLnQ6XSicePGYtCgQeLAgQPV1mk1cuRIAUDExMRUem3Lli3i4YcfFiEhIUKr1YqQkBAxfPhwceLEiWrf92ZTwa//72qdOn/69GnRt29fYTAYRFBQkJg1a1alqdx5eXliypQpIiQkRHh4eIg77rhDvPPOO3ZTvK0+++wz2/UNCAgQvXr1Eps3b7arb+DAgZXO69Wrl+jVq1e134/I1SQhatE/A4nILQwePBhHjhzByZMnlS7FrYwePRqrVq2qdgA0UX3HMTdEdFuKiorstk+ePIn169fj/vvvV6YgIqr3OOaGiG5Lq1atMHr0aLRq1Qrnz5/H4sWLodVq8dJLLyldGhHVUww3RHRb+vfvj2+++QYZGRnQ6XSIjo7G3LlzcccddyhdGhHVUxxzQ0RERG6FY26IiIjIrTDcEBERkVupd2NuzGYzLl68CB8fn0qr3hIREVHtJIRAXl4eQkJCKq1i/2f1LtxcvHgRoaGhSpdBREREtyA1NRXNmjW76TH1Ltz4+PgAsFwcX19fhashIiKimsjNzUVoaKjt7/jN1LtwY+2K8vX1ZbghIiKqY2oypIQDiomIiMitMNwQERGRW2G4ISIiIrfCcENERERuheGGiIiI3ArDDREREbkVhhsiIiJyKww3RERE5FYYboiIiMitMNwQERGRW2G4ISIiIrfCcENERERuheHGSUxmgczcYpzLLlC6FCIionqN4cZJ0nOKEDV3C/q+v1PpUoiIiOo1hhsn8dZpAADGMjNKTWaFqyEiIqq/GG6cxKDV2H4uLDEpWAkREVH9xnDjJFqNClq15XLmG8sUroaIiKj+YrhxIoNODQAoLGG4ISIiUgrDjRN5lXdN5TPcEBERKUbRcLNz507ExcUhJCQEkiRhzZo11Z7z1VdfISIiAgaDAcHBwRg7diwuX74sf7E14GVtuTFyzA0REZFSFA03BQUFiIiIwKJFi2p0/O7duzFq1Cg8+eSTOHLkCFauXIl9+/bh6aeflrnSmvHSseWGiIhIaZrqD5FPbGwsYmNja3z8nj17EBYWhkmTJgEAWrZsib///e94++235SrRIdZuqUIOKCYiIlJMnRpzEx0djdTUVKxfvx5CCGRmZmLVqlUYMGDADc8pKSlBbm6u3UMu1m6pAk4FJyIiUkydCjc9evTAV199hWHDhkGr1aJJkybw8/O7abdWYmIi/Pz8bI/Q0FDZ6rO23BSwW4qIiEgxdSrc/PHHH5g8eTJmzpyJpKQkbNy4EefOncO4ceNueM706dORk5Nje6SmpspWn3XMTQEHFBMRESlG0TE3jkpMTESPHj3w4osvAgA6deoELy8v9OzZE2+88QaCg4MrnaPT6aDT6VxSn8HWLcWWGyIiIqXUqZabwsJCqFT2JavVlkAhhFCiJDveHFBMRESkOEXDTX5+PpKTk5GcnAwAOHv2LJKTk5GSkgLA0qU0atQo2/FxcXH47rvvsHjxYpw5cwa7d+/GpEmT0L17d4SEhCjxFewYbFPB2S1FRESkFEW7pQ4cOIDevXvbthMSEgAA8fHxWLp0KdLT021BBwBGjx6NvLw8LFy4EFOnToW/vz8eeOCBWjMV3JvLLxARESlOErWhP8eFcnNz4efnh5ycHPj6+jr1vX/49SImfnMIUS0bYPnfo5363kRERPWZI3+/69SYm9rOW2cdc8NuKSIiIqUw3DiRQcvZUkREREpjuHGiivvcMNwQEREpheHGiWzhhrOliIiIFMNw40S2taWMZbXivjtERET1EcONE1nXlhICKCpl6w0REZESGG6cyNNDDUmy/MyuKSIiImUw3DiRSiXB4MEZU0REREpiuHEyzpgiIiJSFsONk3HGFBERkbIYbpzs+hlTRERE5HoMN05m0FpbbhhuiIiIlMBw42S29aXYLUVERKQIhhsns64vlc+WGyIiIkUw3DhZxcrgDDdERERKYLhxMuuYm3x2SxERESmC4cbJvMtnS7HlhoiISBkMN05m0FlbbhhuiIiIlMBw42RenC1FRESkKIYbJ/PS8iZ+RERESmK4cbKK5RcYboiIiJTAcONkXlquLUVERKQkhhsn49pSREREymK4cTJ2SxERESmL4cbJbOHGyG4pIiIiJTDcOJl1tpSxzIxSk1nhaoiIiOofhhsnsy6/APBeN0REREpguHEyrUYFrdpyWfM5qJiIiMjlGG5kYJ0xVchBxURERC7HcCODipXBGW6IiIhcjeFGBt7W9aU4Y4qIiMjlGG5kYCjvlmLLDRERkespGm527tyJuLg4hISEQJIkrFmzptpzSkpKMGPGDLRo0QI6nQ5hYWH47LPP5C/WARUtNww3RERErqap/hD5FBQUICIiAmPHjsUjjzxSo3Mee+wxZGZm4tNPP0Xr1q2Rnp4Os7l23U/GoLW23LBbioiIyNUUDTexsbGIjY2t8fEbN27Ejh07cObMGTRo0AAAEBYWJlN1t856l2LOliIiInK9OjXm5vvvv0fXrl0xb948NG3aFG3atMELL7yAoqKiG55TUlKC3Nxcu4fcKlYGZ7ghIiJyNUVbbhx15swZ7Nq1C3q9HqtXr0Z2djaee+45XL58GZ9//nmV5yQmJmL27NkurZPrSxERESmnTrXcmM1mSJKEr776Ct27d8eAAQMwf/58fPHFFzdsvZk+fTpycnJsj9TUVNnrtK4vxZYbIiIi16tTLTfBwcFo2rQp/Pz8bPvatWsHIQQuXLiAO+64o9I5Op0OOp3OlWWy5YaIiEhBdarlpkePHrh48SLy8/Nt+06cOAGVSoVmzZopWJk9Lr9ARESkHEXDTX5+PpKTk5GcnAwAOHv2LJKTk5GSkgLA0qU0atQo2/EjRoxAw4YNMWbMGPzxxx/YuXMnXnzxRYwdOxaenp5KfIUqWVtueBM/IiIi11M03Bw4cACRkZGIjIwEACQkJCAyMhIzZ84EAKSnp9uCDgB4e3tj8+bNuHbtGrp27YqRI0ciLi4OH3zwgSL134h1thSXXyAiInI9Rcfc3H///RBC3PD1pUuXVtp35513YvPmzTJWdftsY27YckNERORydWrMTV1hvUNxAZdfICIicjmGGxl421pu2C1FRETkagw3MrCuCl5gLLtptxsRERE5H8ONDKwDioUAikrZekNERORKDDcy8PRQQ5IsP7NrioiIyLUYbmSgUkkweHAJBiIiIiUw3MikYgkGhhsiIiJXYriRiRdnTBERESmC4UYmXjre64aIiEgJDDcyMWh5l2IiIiIlMNzIxHojv0J2SxEREbkUw41MrEswcGVwIiIi12K4kYmt5YZjboiIiFyK4UYm1jE3+eyWIiIicimGG5l4l8+WYssNERGRazHcyMSgs7bcMNwQERG5EsONTLw4W4qIiEgRDDcy8dLyJn5ERERKYLiRScXyCww3RERErsRwIxMvLdeWIiIiUgLDjUy4thQREZEyGG5kwm4pIiIiZTDcyMQWbozsliIiInIlhhuZWGdLGcvMKDWZFa6GiIio/mC4kYl1+QWA97ohIiJyJYYbmWg1KmjVlsubz0HFRERELsNwIyPrjKlCDiomIiJyGYYbGVWsDM5wQ0RE5CoMNzLytq4vxRlTRERELsNwIyNDebcUW26IiIhch+FGRhUtNww3RERErsJwIyOD1tpyw24pIiIiV1E03OzcuRNxcXEICQmBJElYs2ZNjc/dvXs3NBoNOnfuLFt9t8t6l2LOliIiInIdRcNNQUEBIiIisGjRIofOu3btGkaNGoU+ffrIVJlzVKwMznBDRETkKprqD5FPbGwsYmNjHT5v3LhxGDFiBNRqtUOtPa7G9aWIiIhcr86Nufn8889x5swZzJo1q0bHl5SUIDc31+7hKtb1pdhyQ0RE5Dp1KtycPHkS06ZNw5dffgmNpmaNTomJifDz87M9QkNDZa6yAltuiIiIXK/OhBuTyYQRI0Zg9uzZaNOmTY3Pmz59OnJycmyP1NRUGau0Z11+gS03RERErqPomBtH5OXl4cCBAzh06BAmTJgAADCbzRBCQKPRYNOmTXjggQcqnafT6aDT6VxdLoDrWm4YboiIiFymzoQbX19fHD582G7fhx9+iK1bt2LVqlVo2bKlQpXdmG22FG/iR0RE5DKKhpv8/HycOnXKtn327FkkJyejQYMGaN68OaZPn460tDT85z//gUqlQseOHe3Ob9y4MfR6faX9tUXFfW445oaIiMhVFA03Bw4cQO/evW3bCQkJAID4+HgsXboU6enpSElJUaq821Zxh2K23BAREbmKJIQQShfhSrm5ufDz80NOTg58fX1l/axz2QW4/93t8NZp8PvsfrJ+FhERkTtz5O93nZktVRdZVwUvMJahnmVIIiIixTDcyMi6KrgQQFEpx90QERG5AsONjDw91JAky88FHFRMRETkEgw3MpIkiYtnEhERuRjDjcysM6Z4rxsiIiLXYLiRmbftLsXsliIiInIFhhuZXT9jioiIiOTHcCMzjrkhIiJyLYYbmXEJBiIiItdiuJGZNdxwCQYiIiLXYLiRmVf5bKlCjrkhIiJyCYYbmVW03LBbioiIyBUYbmTGlhsiIiLXYriRGcfcEBERuRbDjcwMnC1FRETkUgw3MvPmTfyIiIhciuFGZgbexI+IiMilGG5kxrWliIiIXIvhRmZcFZyIiMi1GG5kVtFyw3BDRETkCgw3MrPOliowsluKiIjIFRhuZOZdPqDYWGZGqcmscDVERETuj+FGZobyqeAA73VDRETkCg6Hmy+++ALr1q2zbb/00kvw9/fHPffcg/Pnzzu1OHfgoVZBq7Fc5nwOKiYiIpKdw+Fm7ty58PT0BADs2bMHixYtwrx58xAYGIgpU6Y4vUB3YFtfioOKiYiIZKdx9ITU1FS0bt0aALBmzRoMHToUzzzzDHr06IH777/f2fW5BS+dBlcLS7m+FBERkQs43HLj7e2Ny5cvAwA2bdqEBx98EACg1+tRVFTk3OrchFf5oOJCzpgiIiKSncMtNw8++CCeeuopREZG4sSJExgwYAAA4MiRIwgLC3N2fW7Bq3xQMVtuiIiI5Odwy82iRYsQHR2NS5cu4dtvv0XDhg0BAElJSRg+fLjTC3QHXtaVwTmgmIiISHYOt9z4+/tj4cKFlfbPnj3bKQW5I2u3VD6nghMREcnO4ZabjRs3YteuXbbtRYsWoXPnzhgxYgSuXr3q1OLchfVeN5wtRUREJD+Hw82LL76I3NxcAMDhw4cxdepUDBgwAGfPnkVCQoJD77Vz507ExcUhJCQEkiRhzZo1Nz3+u+++w4MPPohGjRrB19cX0dHR+Omnnxz9Ci7H9aWIiIhcx+Fwc/bsWbRv3x4A8O2332LQoEGYO3cuFi1ahA0bNjj0XgUFBYiIiMCiRYtqdPzOnTvx4IMPYv369UhKSkLv3r0RFxeHQ4cOOfo1XMqg5fpSREREruLwmButVovCwkIAwM8//4xRo0YBABo0aGBr0amp2NhYxMbG1vj4999/32577ty5WLt2LX744QdERkY69Nmu5F3eLcWWGyIiIvk5HG7uvfdeJCQkoEePHti3bx+WL18OADhx4gSaNWvm9AJvxmw2Iy8vDw0aNHDp5zqKLTdERESu43C31MKFC6HRaLBq1SosXrwYTZs2BQBs2LAB/fv3d3qBN/Puu+8iPz8fjz322A2PKSkpQW5urt3D1TjmhoiIyHUcbrlp3rw5fvzxx0r7FyxY4JSCaurrr7/G7NmzsXbtWjRu3PiGxyUmJio+Td3AbikiIiKXcTjcAIDJZMKaNWtw9OhRAECHDh3w0EMPQa1WO7W4G1m2bBmeeuoprFy5EjExMTc9dvr06XazuHJzcxEaGip3iXasN/Er4E38iIiIZOdwuDl16hQGDBiAtLQ0tG3bFoCldSQ0NBTr1q1DeHi404u83jfffIOxY8di2bJlGDhwYLXH63Q66HQ6WWuqjm1tKd7Ej4iISHYOj7mZNGkSwsPDkZqaioMHD+LgwYNISUlBy5YtMWnSJIfeKz8/H8nJyUhOTgZgmWaenJyMlJQUAJZWF+tsLMDSFTVq1Ci89957iIqKQkZGBjIyMpCTk+Po13Apri1FRETkOg6Hmx07dmDevHl2M5QaNmyIt956Czt27HDovQ4cOIDIyEjbNO6EhARERkZi5syZAID09HRb0AGAjz76CGVlZRg/fjyCg4Ntj8mTJzv6NVyKq4ITERG5jsPdUjqdDnl5eZX25+fnQ6vVOvRe999/P4QQN3x96dKldtvbt2936P1ri+vH3AghIEmSwhURERG5L4dbbgYNGoRnnnkGv/zyC4QQEEJg7969GDduHB566CE5aqzzrN1SQgBFpWy9ISIikpPD4eaDDz5AeHg4oqOjodfrodfr0aNHD7Ru3brSHYTJwtNDDWtjDcfdEBERycvhbil/f3+sXbsWp06dsk0Fb9euHVq3bu304tyFJEnw0mqQX1JmmTHlo3RFRERE7uuW7nMDAK1bt7YLNL/99hu6du0Ko9HolMLcjZdOjfySMrbcEBERyczhbqkbEULAZOJ4khvhjCkiIiLXcFq4oZvjXYqJiIhcg+HGRQxari9FRETkCjUec1PdatpV3fuGKlhXBucSDERERPKqcbjx9/e/6c3neHO6mzOUhxsOKCYiIpJXjcPNtm3b5KzD7XmX38ivkGNuiIiIZFXjcNOrVy8563B7Bq215YbdUkRERHLigGIXsc6WYssNERGRvBhuXMSrfLYUx9wQERHJi+HGRbw4W4qIiMglGG5cxLoyOG/iR0REJC+GGxexDijmTfyIiIjk5fDCmUOGDKnyfjaSJEGv16N169YYMWIE2rZt65QC3YX1Jn4F7JYiIiKSlcMtN35+fti6dSsOHjwISZIgSRIOHTqErVu3oqysDMuXL0dERAR2794tR711lm35BXZLERERycrhlpsmTZpgxIgRWLhwIVQqSzYym82YPHkyfHx8sGzZMowbNw4vv/wydu3a5fSC66qKlhuGGyIiIjk53HLz6aef4vnnn7cFGwBQqVSYOHEiPvroI0iShAkTJuD33393aqF1ncG2Kji7pYiIiOTkcLgpKyvDsWPHKu0/duwYTCbLH269Xs91pv7Eu3xAsbHMjFKTWeFqiIiI3JfD3VJPPPEEnnzySbzyyivo1q0bAGD//v2YO3cuRo0aBQDYsWMHOnTo4NxK6zhD+VRwwHKvGz8DJ6oRERHJweFws2DBAgQFBWHevHnIzMwEAAQFBWHKlCl4+eWXAQB9+/ZF//79nVtpHeehVkGrUcFYZka+sQx+Bg+lSyIiInJLDocbtVqNGTNmYMaMGcjNzQUA+Pr62h3TvHlz51TnZry0ahjLzCjkoGIiIiLZOBxurvfnUEM356XT4GphKdeXIiIikpHDAz8yMzPxxBNPICQkBBqNBmq12u5BN+alta4MzhlTREREcnG45Wb06NFISUnBa6+9huDgYM6KcoB1fSm23BAREcnH4XCza9cu/O9//0Pnzp1lKMe92VYG512KiYiIZONwt1RoaCiEEHLU4vas3VL5XF+KiIhINg6Hm/fffx/Tpk3DuXPnZCjHvVnvdcPZUkRERPJxuFtq2LBhKCwsRHh4OAwGAzw87O/XcuXKFacV5264vhQREZH8HA4377//vgxl1A8GLdeXIiIikpvD4SY+Pt5pH75z50688847SEpKQnp6OlavXo3Bgwff9Jzt27cjISEBR44cQWhoKF599VWMHj3aaTXJybu8W4otN0RERPKp0Zgb652IrT/f7OGIgoICREREYNGiRTU6/uzZsxg4cCB69+6N5ORkPP/883jqqafw008/OfS5SmHLDRERkfxq1HITEBCA9PR0NG7cGP7+/lXe20YIAUmSbCuD10RsbCxiY2NrfPySJUvQsmVLvPfeewCAdu3aYdeuXViwYAH69etX4/dRCsfcEBERya9G4Wbr1q1o0KABAGDbtm2yFnQze/bsQUxMjN2+fv364fnnn1emIAcZ2C1FREQkuxqFm169elX5s6tlZGQgKCjIbl9QUBByc3NRVFQET0/PSueUlJSgpKTEtu1o15kzWW/iV8Cb+BEREcnmlhbOvHbtGvbt24esrCyYzWa710aNGuWUwpwlMTERs2fPVroMANetLcWb+BEREcnG4XDzww8/YOTIkcjPz4evr6/d+BtJkmQNN02aNEFmZqbdvszMTPj6+lbZagMA06dPR0JCgm07NzcXoaGhstV4M1xbioiISH4Oh5upU6di7NixmDt3LgwGgxw13VB0dDTWr19vt2/z5s2Ijo6+4Tk6nQ46nU7u0mqEq4ITERHJz+HlF9LS0jBp0iSnBJv8/HwkJycjOTkZgGWqd3JyMlJSUgBYWl2ubwkaN24czpw5g5deegnHjh3Dhx9+iBUrVmDKlCm3XYsrXD/mhutzERERycPhcNOvXz8cOHDAKR9+4MABREZGIjIyEgCQkJCAyMhIzJw5EwCQnp5uCzoA0LJlS6xbtw6bN29GREQE3nvvPXzyySd1Yho4UNEtJQRQVMrWGyIiIjk43C01cOBAvPjii/jjjz/wl7/8pdLaUg899FCN3+v++++/aQvG0qVLqzzn0KFDNf6M2sTTQw1JsoSb/JIy2039iIiIyHkc/uv69NNPAwDmzJlT6TVHb+JX30iSBC+tBvklZZYZUz5KV0REROR+HA43f576TY7x1lnCTW5xqdKlEBERuSWHx9zQ7Qn21wMALl4rUrgSIiIi91SjlpsPPvgAzzzzDPR6PT744IObHjtp0iSnFOauQgMMOJRyDalXGG6IiIjkUKNws2DBAowcORJ6vR4LFiy44XGSJDHcVKNZgOVmgxeuFipcCRERkXuqUbg5e/ZslT+T40IbWO4PlHqVLTdERERy4JgbF2PLDRERkbxu6UYrFy5cwPfff4+UlBQYjUa71+bPn++UwtxVaEB5y82VIggh7NbmIiIiotvncLjZsmULHnroIbRq1QrHjh1Dx44dce7cOQghcNddd8lRo1sJ9tdDkix3KL5SYERD79qx7hUREZG7cLhbavr06XjhhRdw+PBh6PV6fPvtt0hNTUWvXr3w17/+VY4a3YpOo0YTX8t0cI67ISIicj6Hw83Ro0dti1lqNBoUFRXB29sbc+bMwdtvv+30At0Rx90QERHJx+Fw4+XlZRtnExwcjNOnT9tey87Odl5lbuz6cTdERETkXA6Pubn77ruxa9cutGvXDgMGDMDUqVNx+PBhfPfdd7j77rvlqNHtsOWGiIhIPg6Hm/nz5yM/Px8AMHv2bOTn52P58uW44447OFOqhprxXjdERESycSjcmEwmXLhwAZ06dQJg6aJasmSJLIW5M7bcEBERycehMTdqtRp9+/bF1atX5aqnXrCOublwtQhms1C4GiIiIvfi8IDijh074syZM3LUUm8E++mhVkkwlplxKb9E6XKIiIjcisPh5o033sALL7yAH3/8Eenp6cjNzbV7UPU0ahWC/Sz3umHXFBERkXPVONzMmTMHBQUFGDBgAH799Vc89NBDaNasGQICAhAQEAB/f38EBATIWatbsY674XRwIiIi56rxgOLZs2dj3Lhx2LZtm5z11BuhAQbsxRW23BARETlZjcONEJaBr7169ZKtmPqkGW/kR0REJAuHxtxwBWvnCW1QPh38GltuiIiInMmh+9y0adOm2oBz5cqV2yqovmDLDRERkTwcCjezZ8+Gn5+fXLXUK9aWm4vXimAyC6hVbBUjIiJyBofCzeOPP47GjRvLVUu90thHDw+1hFKTQEZuMZr6eypdEhERkVuo8ZgbjrdxLrVKsgWaC1c47oaIiMhZahxurLOlyHls4264gCYREZHT1Lhbymw2y1lHvWSbMcV73RARETmNw8svkPNwxhQREZHzMdwoyLoEA1tuiIiInIfhRkHWlpsLHHNDRETkNAw3CrKOuUnPKUKpiWOaiIiInKFWhJtFixYhLCwMer0eUVFR2Ldv302Pf//999G2bVt4enoiNDQUU6ZMQXFxsYuqdZ5G3jroNCqYBZB+re7VT0REVBspHm6WL1+OhIQEzJo1CwcPHkRERAT69euHrKysKo//+uuvMW3aNMyaNQtHjx7Fp59+iuXLl+OVV15xceW3T5IkjrshIiJyMsXDzfz58/H0009jzJgxaN++PZYsWQKDwYDPPvusyuP/7//+Dz169MCIESMQFhaGvn37Yvjw4dW29tRWFfe6YbghIiJyBkXDjdFoRFJSEmJiYmz7VCoVYmJisGfPnirPueeee5CUlGQLM2fOnMH69esxYMCAKo8vKSlBbm6u3aM2qbjXDQcVExEROYNDa0s5W3Z2NkwmE4KCguz2BwUF4dixY1WeM2LECGRnZ+Pee++FEAJlZWUYN27cDbulEhMTMXv2bKfX7iwV97phyw0REZEzKN4t5ajt27dj7ty5+PDDD3Hw4EF89913WLduHf7xj39Uefz06dORk5Nje6Smprq44psL5XRwIiIip1K05SYwMBBqtRqZmZl2+zMzM9GkSZMqz3nttdfwxBNP4KmnngIA/OUvf0FBQQGeeeYZzJgxAyqVfV7T6XTQ6XTyfAEnsA4o5pgbIiIi51C05Uar1aJLly7YsmWLbZ/ZbMaWLVsQHR1d5TmFhYWVAoxarQZQNxf3DG1gabnJzC1BSZlJ4WqIiIjqPkVbbgAgISEB8fHx6Nq1K7p37473338fBQUFGDNmDABg1KhRaNq0KRITEwEAcXFxmD9/PiIjIxEVFYVTp07htddeQ1xcnC3k1CUBBg8YtGoUGk1Iu1qEVo28lS6JiIioTlM83AwbNgyXLl3CzJkzkZGRgc6dO2Pjxo22QcYpKSl2LTWvvvoqJEnCq6++irS0NDRq1AhxcXF48803lfoKt0WSJIQGGHA8Mw8XGG6IiIhumyTqYl/ObcjNzYWfnx9ycnLg6+urdDkAgCeX7seWY1l4c0hHjIxqoXQ5REREtY4jf7/r3Gwpd2Qdd8MZU0RERLeP4aYWsM2Y4r1uiIiIbhvDTS3QjPe6ISIichqGm1qAi2cSERE5D8NNLWAdc5Odb0SRkfe6ISIiuh0MN7WAn6cHfPSWWflsvSEiIro9DDe1BNeYIiIicg6Gm1qCa0wRERE5B8NNLcF73RARETkHw00twXvdEBEROQfDTS3BMTdERETOwXBTSzRrwDE3REREzsBwU0tY71J8rbAUecWlCldDRERUdzHc1BLeOg0CDB4A2DVFRER0OxhuahHOmCIiIrp9DDe1CGdMERER3T6Gm1qEM6aIiIhuH8NNLcK7FBMREd0+hptapBnH3BAREd02hptaJLS85ebClUIIIRSuhoiIqG5iuKlFrPe6ySspQ25RmcLVEBER1U0MN7WI3kONQG8dAI67ISIiulUMN7VMaPkyDBcYboiIiG4Jw00tY+2aSr3CQcVERES3guGmlrENKmbLDRER0S1huKllbC03nA5ORER0SxhuahmOuSEiIro9DDe1zPVjbnivGyIiIscx3NQyIf56SBJQVGrClQKj0uUQERHVOQw3tYxOo0aQjx4Ax90QERHdCoabWsg67ub85QKFKyEiIqp7GG5qoQ4hfgCAbceyFK6EiIio7qkV4WbRokUICwuDXq9HVFQU9u3bd9Pjr127hvHjxyM4OBg6nQ5t2rTB+vXrXVSt/B7uHAIA+OlIJgpKuMYUERGRIxQPN8uXL0dCQgJmzZqFgwcPIiIiAv369UNWVtWtFkajEQ8++CDOnTuHVatW4fjx4/j444/RtGlTF1cun86h/mgZ6IWiUhN+OpKhdDlERER1iuLhZv78+Xj66acxZswYtG/fHkuWLIHBYMBnn31W5fGfffYZrly5gjVr1qBHjx4ICwtDr169EBER4eLK5SNJEoZEWsLa6kNpCldDRERUtygaboxGI5KSkhATE2Pbp1KpEBMTgz179lR5zvfff4/o6GiMHz8eQUFB6NixI+bOnQuTyVTl8SUlJcjNzbV71AWDO1vCze5T2cjMLVa4GiIiorpD0XCTnZ0Nk8mEoKAgu/1BQUHIyKi6O+bMmTNYtWoVTCYT1q9fj9deew3vvfce3njjjSqPT0xMhJ+fn+0RGhrq9O8hh+YNDejaIgBmAaxNZusNERFRTSneLeUos9mMxo0b46OPPkKXLl0wbNgwzJgxA0uWLKny+OnTpyMnJ8f2SE1NdXHFt27IXZbWm+8OMtwQERHVlKLhJjAwEGq1GpmZmXb7MzMz0aRJkyrPCQ4ORps2baBWq2372rVrh4yMDBiNle/oq9Pp4Ovra/eoKwb9JQRatQrHMvJwNL1udKcREREpTdFwo9Vq0aVLF2zZssW2z2w2Y8uWLYiOjq7ynB49euDUqVMwm822fSdOnEBwcDC0Wq3sNbuSn8EDD9zZGACwhgOLiYiIakTxbqmEhAR8/PHH+OKLL3D06FE8++yzKCgowJgxYwAAo0aNwvTp023HP/vss7hy5QomT56MEydOYN26dZg7dy7Gjx+v1FeQ1eDyWVNrktNgMnMhTSIioupolC5g2LBhuHTpEmbOnImMjAx07twZGzdutA0yTklJgUpVkcFCQ0Px008/YcqUKejUqROaNm2KyZMn4+WXX1bqK8iq952N4OfpgczcEuw5fRn33hGodElERES1miSEqFfNAbm5ufDz80NOTk6dGX8zY/VhfPVLCh65qynmP9ZZ6XKIiIhczpG/34p3S1H1HimfNbXx9wwUGrkcAxER0c0w3NQBdzUPQPMGBhQaTdh0JLP6E4iIiOoxhps6QJIk28BiLsdARER0cww3dYR1ran/nbyErDwux0BERHQjDDd1RMtAL0Q294dZAN8nX1S6HCIiolqL4aYOeYRdU0RERNViuKlDBnYKgUYl4cjFXJzIzFO6HCIiolqJ4aYOaeClxf1tLcsxcDFNIiKiqjHc1DHWe96sTU6DmcsxEBERVcJwU8c8cGdj+Og1SM8pxt6zl5Uuh4iIqNZhuKlj9B5qDOoUDABYza4pIiKiShhu6qDBnS1dUxt+z0CR0aRwNURERLULw00d1C2sAZr6eyK/pAyb/shQuhwiIqJaheGmDlKpJAwtH1g8b+NxXC0wKlwRERFR7cFwU0c9fV8rtAz0Qtq1IiSsSObMKSIionIMN3WUj94Di0bcBa1GhW3HL+HfO88oXRIREVGtwHBTh7UP8cWchzoAAN7ddBy/nOHUcCIiIoabOm5Yt1A8EtkUJrPAxG8OITu/ROmSiIiIFMVwU8dJkoQ3hnTEHY29kZVXgueXJcPE8TdERFSPMdy4AYNWgw9H3gVPDzV2ncrGB1tOKl0SERGRYhhu3MQdQT6Y+0hHAMAHW0/ifycvKVwRERGRMhhu3MiQyGYY3j0UQgDPL0tGRk6x0iURERG5HMONm5kV1wHtgn1xucCIid8cRJnJrHRJRERELsVw42b0Hmp8OPIueOs02H/uKt7ddELpkoiIiFyK4cYNtQz0wttDOwEAluw4jU1HuP4UERHVHww3bmpgp2CMvicMAPDsVwfx0c7TEIJTxImIyP0x3LixVwa0w+DOITCZBeauP4a//zcJOUWlSpdFREQkK4YbN6bVqLBgWGf8Y3BHaNUqbPojEw8t3IUjF3OULo2IiEg2DDduTpIkPHF3C6wcF42m/p44f7kQQz78Pyzfn6J0aURERLJguKknIkL98ePEe9G7bSMYy8x4+dvDeHHlrygympQujYiIyKkYbuqRAC8tPo3vhhf7tYVKAlYmXcCQD3fjbHaB0qURERE5DcNNPaNSSRjfuzW+fDIKgd5aHMvIw0P/2oW1yWmcTUVERG6hVoSbRYsWISwsDHq9HlFRUdi3b1+Nzlu2bBkkScLgwYPlLdAN3dM6ED9O7IluYQHIKynD5GXJiFu4C1uPZTLkEBFRnaZ4uFm+fDkSEhIwa9YsHDx4EBEREejXrx+ysrJuet65c+fwwgsvoGfPni6q1P008dPj66fvxuQ+d8CgVeP3tFyMXXoAQz78P+w8cYkhh4iI6iRJKPwXLCoqCt26dcPChQsBAGazGaGhoZg4cSKmTZtW5Tkmkwn33Xcfxo4di//973+4du0a1qxZU6PPy83NhZ+fH3JycuDr6+usr1HnXc4vwUc7z+CLPedQXGpZj6pbWACmPNgG94QHKlwdERHVd478/Va05cZoNCIpKQkxMTG2fSqVCjExMdizZ88Nz5szZw4aN26MJ598strPKCkpQW5urt2DKmvorcP0Ae2w86XeGNMjDFqNCvvPXcWIj3/B8I/24sC5K0qXSEREVCOKhpvs7GyYTCYEBQXZ7Q8KCkJGRtXrIe3atQuffvopPv744xp9RmJiIvz8/GyP0NDQ267bnTX20WNWXAfsfLE3nri7BTzUEvacuYxHl+zByE/2Ym1yGqePExFRrab4mBtH5OXl4YknnsDHH3+MwMCadZVMnz4dOTk5tkdqaqrMVbqHJn56/GNwR2x/sTeGdw+FRiVh96nLmLwsGV3f2IypK37FrpPZMJk5LoeIiGoXjZIfHhgYCLVajczMTLv9mZmZaNKkSaXjT58+jXPnziEuLs62z2y2jA/RaDQ4fvw4wsPD7c7R6XTQ6XQyVF8/NPX3ROIjnfDc/a2x8kAqVienIfVKEb49eAHfHryAIF8dBnduisGRTdEumGOYiIhIebViQHH37t3xr3/9C4AlrDRv3hwTJkyoNKC4uLgYp06dstv36quvIi8vD//85z/Rpk0baLXam34eBxTfHiEEks5fxXeH0rDut3S7hTjvbOKDwZFNEdMuCOGNvCBJkoKVEhGRO3Hk77fi4Wb58uWIj4/Hv//9b3Tv3h3vv/8+VqxYgWPHjiEoKAijRo1C06ZNkZiYWOX5o0eP5mwphZSUmbDt2CWsPnQBW49lodRU8avULMAT97dthN5tGyM6vCEMWkUbCYmIqI5z5O+34n9xhg0bhkuXLmHmzJnIyMhA586dsXHjRtsg45SUFKhUdWpoUL2h06jRv2MT9O/YBNcKjVh3OB0bDmdg39kruHC1CF/uTcGXe1OgVasQ1aoBerVphPvbNmarDhERyUrxlhtXY8uN/ApKyrDn9GVsP5GF7ccv4cLVIrvXQxt4ontYQ0Q298ddzQPQtokP1CqGHSIiurE61S3lagw3riWEwOlL+dh+/BK2H7+EfWevwGgy2x1j0KoR0cwfd7XwR2RoACKb+6OhNweBExFRBYabm2C4UVZBSRn2nbuCQ+ev4mDKNSSnXkN+SVml41o0NKBTM3+0D/ZF+xBftA/2RSMfBh4iovqK4eYmGG5qF5NZ4FRWPg6mXMXB81dxKPUaTmXlV3lsYx+dLehYn8MaekHFLi0iIrfHcHMTDDe1X05hKZIvXMORizn442Iu/kjPxdnsAlT1m6rTqNAy0AutGnmhVaC35bmRN1oGesHP08P1xRMRkSwYbm6C4aZuKigpw7GMPPyRnmsLPMfSc1FSZr7hOYHeWrQKtASd5g0NaNHQgOYNDGjRwAt+BgYfIqK6hOHmJhhu3EeZyYy0a0U4c6kApy/l40x2Ac5cyseZSwXIyiu56bl+nh5o3sCA5rbAY0DTAE+E+Huiqb8n9B5qF30LIiKqCYabm2C4qR/yiktxNrsAZy4V4Gx2AVKvFOL8lUKkXCnEpWqCD2Bp9bEGnab+nrbgE+ynR5CvHoHeOk5fJyJyIYabm2C4oUJjGVKvFOH85QKklAeelCuFSLtahLRrRSiswarnapWERt46BPnqEOSrR5Py0BPkq0djHx0CvXUI9NGioRdDEBGRM9SpOxQTuZpBq0HbJj5o28Sn0mtCCOQUleLC1SJcvGYJO2lXi3Axx/KckVuMS3klMJkFMnKLkZFbDCDnhp8lSUADgxaB3jo08tEh0Nvyc0NvHQIMHgjw0iLAoEUDLw/4G7Tw9/SARs07chMR3Q6GG6LrSJJkCRkGLTo29avyGJNZIDu/BBk5xcjMtTwycouRmVuCzPLwk51fgssFRggBXC4w4nKBEccz82pUg69eYws9fp4e8Dd4WJ49PeBn3efpAT+D5dm/fJ9Ww1BERAQw3BA5TK2SbF1QN2MyC1wpMCI7v8QWeCwPIy7nG3Gt0IgrhUZcKyzFlQKjbYX13OIy5BaX4fzlQofq8tZp4OfpgQAvD/h7auFv8ECAQWsLR756D/h6asqfLdt+nh7w1mvYdUZEboXhhkgmapWERj6W7qh2wdUfX2YyI6eoFFcLS3G10Iir5YHH+rhWWIpr1u1Co+3Y3OJSCAHkl5Qhv6QMadeKqv+wP/HRaeCj18Bbr4G3TgNvvQd8dNafy18r3/bSVRznpa04xkunhk7DWWZEpDyGG6JaQqNWoWH5eBxHmMwCecUVoSin/PlaYSmuFRpxtbAUecWllhah8nCUW1yK3KIyFJVaBk/nlZQhr6TsZsOHasRDLVUEIN31wUcDb23Fzz46DQw6Nby0Ghi0ahi09tteOsuzTqPiCvJE5DCGG6I6Tq2qGCfUEl4OnWssMyOv2BJ4CkpMyCsuRV5JGfKLy2wtQXnFZcgvKUVecRkKyrcLjGUoKDFZjimuCEmlJlEeskqd8t0kCdBr1NB7qODpoYbe9lDBU6u2vFb+7KlVlT//6TgPteVhDVFay89eWk35PjU8OIibyK0w3BDVY1rNrbUW/ZnJLMoDj30wsoWhkjIUGE22n/NLylBoLEOh0YSCkvJnYxkKSyzPxaWWO08LARSVmlBUasJVOCcwVcVDLdkFIkugKv/Zw9Ld5lnekqTVqKBVW5491BK0ajU8NNJ1+1SVwpin3bMKeut7qdkyRSQHhhsium1qlWQZqKz3AKqeZOYQk1mg0GhpESopNaOo1ITiUhOKjCYUl5lRfP12qWWf5TUTio2m8uPtzyss328NVUVGE8rMltt8lZoESk2WIOZqWo0KOttDbdvWVrGvYr8lHOk8VNCq1eXPquue1bZtnbr64zzUEkMWuRWGGyKqddQqCT56D/jo5V0DzFhmrgg75UGouNSMklJLUCoylgepMsv+4lITSk3m8oeAscwMo8mMUuuzyQxjmdl27PXvad0uKjXZLQJrLLOcU7MbBchDkmBpeVKr4FHeIuWhVpU/Kv9sbaHSXv+6pmJbYz1eZf3ZcoxGLcFDVf5cfozO+l7WlrDrWsa0GhU0Kmtd5e+hYhCj6jHcEFG9ZfkDqoW/wXWfKYRASXkYKim1Ppv+tG1GSZnJFp6s2yVl5useJluQqjjuT+9TfkxJeYCqeDah1CSuqwm290X1q5Mozj5wqaBVS+Wh7Lpt6+say7ZGpYJaLUGjkqBWWZ9VUKtgeU1l/5rqumNs+9WWZ2uw02oqQp/WFu5UtiBney+p4ly1ZF+L9b0Z2JyL4YaIyIUkqWJ8D25+qyRZmc2iPABVhCBL91xFy1RpeatUqVnYtU5df9yfzzOW7yszCZSZLa+VXXdOmVlUav2yvo/R9n4V29eHMCvLe5kAVL9USl3hUR7ArK1aapVka/myBiBrsLILYmpLAPO4bltjF8hU9oHOFqpUdttqlQpqCVCrVVBLEtQq2MKf5bWKfZrrPseuVa78s6wtcdXdC0xODDdERPWQSiVBryoPWZC3++92CCFQZhYoM4nrwpXZtm0NUte/ZglH4rrwZQlVJrMofy7fNgnbfpOwhDCTGRWvm0Wl8+wCYJlAiTUAmq7vorQca7ruva3vcaPVHG2BTb5x8y4V6K3DgVdjFPt8hhsiIqq1JEkqbx0APFH3bxJp/lPYsbZqlZktIe361q0y0/XhylwRsv60v8x03ftdt11qNtuOLasi0NkFvfJt85+fhSVYXl+z6U+1VlW3p1bZ2ysw3BAREbmISiVBBUtYI/nwzlVERETkVhhuiIiIyK0w3BAREZFbYbghIiIit8JwQ0RERG6F4YaIiIjcCsMNERERuRWGGyIiInIrDDdERETkVhhuiIiIyK3UinCzaNEihIWFQa/XIyoqCvv27bvhsR9//DF69uyJgIAABAQEICYm5qbHExERUf2ieLhZvnw5EhISMGvWLBw8eBARERHo168fsrKyqjx++/btGD58OLZt24Y9e/YgNDQUffv2RVpamosrJyIiotpIEuJGC7C7RlRUFLp164aFCxcCAMxmM0JDQzFx4kRMmzat2vNNJhMCAgKwcOFCjBo1qtrjc3Nz4efnh5ycHPj6+t52/URERCQ/R/5+K9pyYzQakZSUhJiYGNs+lUqFmJgY7Nmzp0bvUVhYiNLSUjRo0KDK10tKSpCbm2v3ICIiIvelUfLDs7OzYTKZEBQUZLc/KCgIx44dq9F7vPzyywgJCbELSNdLTEzE7NmzK+1nyCEiIqo7rH+3a9LhpGi4uV1vvfUWli1bhu3bt0Ov11d5zPTp05GQkGDbTktLQ/v27REaGuqqMomIiMhJ8vLy4Ofnd9NjFA03gYGBUKvVyMzMtNufmZmJJk2a3PTcd999F2+99RZ+/vlndOrU6YbH6XQ66HQ627a3tzdSU1Ph4+MDSZJu7wv8SW5uLkJDQ5GamsrxPC7A6+1avN6uxevtWrzernUr11sIgby8PISEhFR7rKLhRqvVokuXLtiyZQsGDx4MwDKgeMuWLZgwYcINz5s3bx7efPNN/PTTT+jatatDn6lSqdCsWbPbKbtavr6+/B+HC/F6uxavt2vxersWr7drOXq9q2uxsVK8WyohIQHx8fHo2rUrunfvjvfffx8FBQUYM2YMAGDUqFFo2rQpEhMTAQBvv/02Zs6cia+//hphYWHIyMgAYGmR8fb2Vux7EBERUe2geLgZNmwYLl26hJkzZyIjIwOdO3fGxo0bbYOMU1JSoFJVTOpavHgxjEYjHn30Ubv3mTVrFl5//XVXlk5ERES1kOLhBgAmTJhww26o7du3222fO3dO/oJukU6nw6xZs+zG+JB8eL1di9fbtXi9XYvX27Xkvt6K38SPiIiIyJkUX36BiIiIyJkYboiIiMitMNwQERGRW2G4ISIiIrfCcOMkixYtQlhYGPR6PaKiorBv3z6lS3IbO3fuRFxcHEJCQiBJEtasWWP3uhACM2fORHBwMDw9PRETE4OTJ08qU2wdl5iYiG7dusHHxweNGzfG4MGDcfz4cbtjiouLMX78eDRs2BDe3t4YOnRopbuMU80sXrwYnTp1st3ILDo6Ghs2bLC9zmstr7feeguSJOH555+37eM1d57XX38dkiTZPe68807b63Jea4YbJ1i+fDkSEhIwa9YsHDx4EBEREejXrx+ysrKULs0tFBQUICIiAosWLary9Xnz5uGDDz7AkiVL8Msvv8DLywv9+vVDcXGxiyut+3bs2IHx48dj79692Lx5M0pLS9G3b18UFBTYjpkyZQp++OEHrFy5Ejt27MDFixfxyCOPKFh13dWsWTO89dZbSEpKwoEDB/DAAw/g4YcfxpEjRwDwWstp//79+Pe//11p+R5ec+fq0KED0tPTbY9du3bZXpP1Wgu6bd27dxfjx4+3bZtMJhESEiISExMVrMo9ARCrV6+2bZvNZtGkSRPxzjvv2PZdu3ZN6HQ68c033yhQoXvJysoSAMSOHTuEEJZr6+HhIVauXGk75ujRowKA2LNnj1JlupWAgADxySef8FrLKC8vT9xxxx1i8+bNolevXmLy5MlCCP5+O9usWbNEREREla/Jfa3ZcnObjEYjkpKSEBMTY9unUqkQExODPXv2KFhZ/XD27FlkZGTYXX8/Pz9ERUXx+jtBTk4OAKBBgwYAgKSkJJSWltpd7zvvvBPNmzfn9b5NJpMJy5YtQ0FBAaKjo3mtZTR+/HgMHDjQ7toC/P2Ww8mTJxESEoJWrVph5MiRSElJASD/ta4Vdyiuy7Kzs2EymWzLRVgFBQXh2LFjClVVf1jXFqvq+ltfo1tjNpvx/PPPo0ePHujYsSMAy/XWarXw9/e3O5bX+9YdPnwY0dHRKC4uhre3N1avXo327dsjOTmZ11oGy5Ytw8GDB7F///5Kr/H327mioqKwdOlStG3bFunp6Zg9ezZ69uyJ33//XfZrzXBDRFUaP348fv/9d7s+cnK+tm3bIjk5GTk5OVi1ahXi4+OxY8cOpctyS6mpqZg8eTI2b94MvV6vdDluLzY21vZzp06dEBUVhRYtWmDFihXw9PSU9bPZLXWbAgMDoVarK43wzszMRJMmTRSqqv6wXmNef+eaMGECfvzxR2zbtg3NmjWz7W/SpAmMRiOuXbtmdzyv963TarVo3bo1unTpgsTEREREROCf//wnr7UMkpKSkJWVhbvuugsajQYajQY7duzABx98AI1Gg6CgIF5zGfn7+6NNmzY4deqU7L/fDDe3SavVokuXLtiyZYttn9lsxpYtWxAdHa1gZfVDy5Yt0aRJE7vrn5ubi19++YXX/xYIITBhwgSsXr0aW7duRcuWLe1e79KlCzw8POyu9/Hjx5GSksLr7SRmsxklJSW81jLo06cPDh8+jOTkZNuja9euGDlypO1nXnP55Ofn4/Tp0wgODpb/9/u2hySTWLZsmdDpdGLp0qXijz/+EM8884zw9/cXGRkZSpfmFvLy8sShQ4fEoUOHBAAxf/58cejQIXH+/HkhhBBvvfWW8Pf3F2vXrhW//fabePjhh0XLli1FUVGRwpXXPc8++6zw8/MT27dvF+np6bZHYWGh7Zhx48aJ5s2bi61bt4oDBw6I6OhoER0drWDVdde0adPEjh07xNmzZ8Vvv/0mpk2bJiRJEps2bRJC8Fq7wvWzpYTgNXemqVOniu3bt4uzZ8+K3bt3i5iYGBEYGCiysrKEEPJea4YbJ/nXv/4lmjdvLrRarejevbvYu3ev0iW5jW3btgkAlR7x8fFCCMt08Ndee00EBQUJnU4n+vTpI44fP65s0XVUVdcZgPj8889txxQVFYnnnntOBAQECIPBIIYMGSLS09OVK7oOGzt2rGjRooXQarWiUaNGok+fPrZgIwSvtSv8OdzwmjvPsGHDRHBwsNBqtaJp06Zi2LBh4tSpU7bX5bzWkhBC3H77DxEREVHtwDE3RERE5FYYboiIiMitMNwQERGRW2G4ISIiIrfCcENERERuheGGiIiI3ArDDREREbkVhhsiIgCSJGHNmjVKl0FETsBwQ0SKGz16NCRJqvTo37+/0qURUR2kUboAIiIA6N+/Pz7//HO7fTqdTqFqiKguY8sNEdUKOp0OTZo0sXsEBAQAsHQZLV68GLGxsfD09ESrVq2watUqu/MPHz6MBx54AJ6enmjYsCGeeeYZ5Ofn2x3z2WefoUOHDtDpdAgODsaECRPsXs/OzsaQIUNgMBhwxx134Pvvv5f3SxORLBhuiKhOeO211zB06FD8+uuvGDlyJB5//HEcPXoUAFBQUIB+/fohICAA+/fvx8qVK/Hzzz/bhZfFixdj/PjxeOaZZ3D48GF8//33aN26td1nzJ49G4899hh+++03DBgwACNHjsSVK1dc+j2JyAmcsvwmEdFtiI+PF2q1Wnh5edk93nzzTSGEZbXycePG2Z0TFRUlnn32WSGEEB999JEICAgQ+fn5ttfXrVsnVCqVyMjIEEIIERISImbMmHHDGgCIV1991badn58vAIgNGzY47XsSkWtwzA0R1Qq9e/fG4sWL7fY1aNDA9nN0dLTda9HR0UhOTgYAHD16FBEREfDy8rK93qNHD5jNZhw/fhySJOHixYvo06fPTWvo1KmT7WcvLy/4+voiKyvrVr8SESmE4YaIagUvL69K3UTO4unpWaPjPDw87LYlSYLZbJajJCKSEcfcEFGdsHfv3krb7dq1AwC0a9cOv/76KwoKCmyv7969GyqVCm3btoWPjw/CwsKwZcsWl9ZMRMpgyw0R1QolJSXIyMiw26fRaBAYGAgAWLlyJbp27Yp7770XX331Ffbt24dPP/0UADBy5EjMmjUL8fHxeP3113Hp0iVMnDgRTzzxBIKCggAAr7/+OsaNG4fGjRsjNjYWeXl52L17NyZOnOjaL0pEsmO4IaJaYePGjQgODrbb17ZtWxw7dgyAZSbTsmXL8NxzzyE4OBjffPMN2rdvDwAwGAz46aefMHnyZHTr1g0GgwFDhw7F/Pnzbe8VHx+P4uJiLFiwAC+88AICAwPx6KOPuu4LEpHLSEIIoXQRREQ3I0kSVq9ejcGDBytdChHVARxzQ0RERG6F4YaIiIjcCsfcEFGtx95zInIEW26IiIjIrTDcEBERkVthuCEiIiK3wnBDREREboXhhoiIiNwKww0RERG5FYYbIiIicisMN0RERORWGG6IiIjIrfw/QPbuassh8/sAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.title(\"Training Loss vs Epoch\")\n",
    "plt.xlabel(\"Epoch\")\n",
    "plt.ylabel(\"Training Loss\")\n",
    "plt.plot(range(epochs), losses)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1622a99c",
   "metadata": {},
   "source": [
    "#### Calculating Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "5fd4c189",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.923\n"
     ]
    }
   ],
   "source": [
    "from collections import defaultdict\n",
    "\n",
    "# Load the test data\n",
    "test_dataset = CustomMNISTDataset('../chapter3/output/testing/')\n",
    "test_dataloader = DataLoader(test_dataset, batch_size=64, shuffle=True)\n",
    "\n",
    "# Set model to eval mode\n",
    "model.eval()\n",
    "\n",
    "# Loop over dataset to compute accuracy\n",
    "correct = 0\n",
    "correct_dict = defaultdict(lambda: [0,0])\n",
    "for batch_idx, (data, targets) in enumerate(test_dataloader):\n",
    "    outputs = model(data)\n",
    "    pred = outputs.argmax(dim=1, keepdim=True) \n",
    "    correct += pred.eq(targets.argmax(dim=2).view_as(pred)).sum().item()\n",
    "    for p, t in zip(pred, targets):\n",
    "        if p[0] == t.argmax():\n",
    "            correct_dict[t.argmax().item()][0] += 1\n",
    "        correct_dict[t.argmax().item()][1] += 1\n",
    "accuracy = correct / test_dataset.__len__()\n",
    "print(f\"Accuracy: {accuracy}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ed745238",
   "metadata": {},
   "source": [
    "### Per-Class Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "d5a9cd1b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(5, 0.8632286995515696),\n",
       " (8, 0.8880903490759754),\n",
       " (2, 0.8982558139534884),\n",
       " (9, 0.9018830525272548),\n",
       " (3, 0.907920792079208),\n",
       " (7, 0.914396887159533),\n",
       " (4, 0.9317718940936863),\n",
       " (6, 0.9561586638830898),\n",
       " (1, 0.9770925110132158),\n",
       " (0, 0.9806122448979592)]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c = [(k, v[0]/v[1]) for k, v in correct_dict.items()]\n",
    "c.sort(key=lambda x: x[1])\n",
    "c"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4b97e779",
   "metadata": {},
   "source": [
    "#### Encapsulating Entire Training and Validation into one Function\n",
    "#### Trainer Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ba11be43",
   "metadata": {},
   "outputs": [],
   "source": [
    "def trainer(config, epochs=10, data_dir=None):\n",
    "    # Loading Dataset\n",
    "    train_dataset = CustomMNISTDataset(data_dir+'/training/')\n",
    "    train_dataloader = DataLoader(train_dataset, batch_size=config[\"batch_size\"], shuffle=True)\n",
    "\n",
    "    # Initialize Model\n",
    "    model = NeuralNet(config[\"hidden_layer_1\"], config[\"hidden_layer_2\"])\n",
    "    # Initialize Optimizer\n",
    "    optimizer = optim.Adadelta(model.parameters(), lr=config[\"lr\"])\n",
    "    # Loss function\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "    \n",
    "    checkpoint = session.get_checkpoint()\n",
    "\n",
    "    if checkpoint:\n",
    "        checkpoint_state = checkpoint.to_dict()\n",
    "        start_epoch = checkpoint_state[\"epoch\"]\n",
    "        net.load_state_dict(checkpoint_state[\"net_state_dict\"])\n",
    "        optimizer.load_state_dict(checkpoint_state[\"optimizer_state_dict\"])\n",
    "    else:\n",
    "        start_epoch = 0\n",
    "\n",
    "    # set model to train mode\n",
    "    losses = []\n",
    "    model.train()\n",
    "    for epoch in range(epochs):\n",
    "        batch_losses = []    \n",
    "        for batch_idx, (data, targets) in enumerate(train_dataloader):\n",
    "            optimizer.zero_grad()\n",
    "            outputs = model(data)\n",
    "            loss = loss_fn(outputs, targets.squeeze(1).float())\n",
    "            loss.backward() # Backpropagation of loss\n",
    "            optimizer.step() # Updating Weights based on Gradients\n",
    "            batch_losses.append(loss.item())\n",
    "        final_epoch_loss = torch.mean(torch.Tensor(batch_losses))\n",
    "        losses.append(final_epoch_loss.item())\n",
    "        \n",
    "    # Load the test data\n",
    "    test_dataset = CustomMNISTDataset(data_dir+'/testing/')\n",
    "    test_dataloader = DataLoader(test_dataset, batch_size=64, shuffle=True)\n",
    "\n",
    "    # Set model to eval mode\n",
    "    model.eval()\n",
    "\n",
    "    # Loop over dataset to compute accuracy\n",
    "    correct = 0\n",
    "    for batch_idx, (data, targets) in enumerate(test_dataloader):\n",
    "        outputs = model(data)\n",
    "        pred = outputs.argmax(dim=1, keepdim=True) \n",
    "        correct += pred.eq(targets.argmax(dim=2).view_as(pred)).sum().item()\n",
    "    accuracy = correct / test_dataset.__len__()\n",
    "    session.report({\"loss\": sum(losses) / len(losses), \"accuracy\": accuracy})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e47ebfdd",
   "metadata": {},
   "source": [
    "#### Setting Up Ray Tune Configs and Scheduler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "64cc1f17",
   "metadata": {},
   "outputs": [],
   "source": [
    "from functools import partial\n",
    "from ray import tune\n",
    "from ray.air import Checkpoint, session\n",
    "from ray.tune.schedulers import ASHAScheduler\n",
    "\n",
    "max_num_epochs = 10\n",
    "num_samples = 10\n",
    "\n",
    "config = {\n",
    "\t\"hidden_layer_1\": tune.choice([512, 256]),\n",
    "\t\"hidden_layer_2\": tune.choice([128, 64]),\n",
    "\t\"lr\": tune.loguniform(1e-4, 1e-1),\n",
    "\t\"batch_size\": tune.choice([32, 48, 64])}\n",
    "\n",
    "scheduler = ASHAScheduler(\n",
    "        metric=\"loss\",\n",
    "        mode=\"min\",\n",
    "        max_t=max_num_epochs,\n",
    "        grace_period=1,\n",
    "        reduction_factor=2,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b67b9997",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-10-18 21:39:56,322\tINFO worker.py:1642 -- Started a local Ray instance.\n",
      "2023-10-18 21:39:57,151\tINFO tune.py:228 -- Initializing Ray automatically. For cluster usage or custom Ray initialization, call `ray.init(...)` before `tune.run(...)`.\n",
      "2023-10-18 21:39:57,153\tINFO tune.py:645 -- [output] This uses the legacy output and progress reporter, as Jupyter notebooks are not supported by the new engine, yet. For more information, please see https://github.com/ray-project/ray/issues/36949\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div class=\"tuneStatus\">\n",
       "  <div style=\"display: flex;flex-direction: row\">\n",
       "    <div style=\"display: flex;flex-direction: column;\">\n",
       "      <h3>Tune Status</h3>\n",
       "      <table>\n",
       "<tbody>\n",
       "<tr><td>Current time:</td><td>2023-10-18 21:42:44</td></tr>\n",
       "<tr><td>Running for: </td><td>00:02:46.98        </td></tr>\n",
       "<tr><td>Memory:      </td><td>7.6/8.0 GiB        </td></tr>\n",
       "</tbody>\n",
       "</table>\n",
       "    </div>\n",
       "    <div class=\"vDivider\"></div>\n",
       "    <div class=\"systemInfo\">\n",
       "      <h3>System Info</h3>\n",
       "      Using AsyncHyperBand: num_stopped=3<br>Bracket: Iter 8.000: None | Iter 4.000: None | Iter 2.000: None | Iter 1.000: -1.4415028870105742<br>Logical resource usage: 1.0/8 CPUs, 0/0 GPUs\n",
       "    </div>\n",
       "    <div class=\"vDivider\"></div>\n",
       "<div class=\"messages\">\n",
       "  <h3>Messages</h3>\n",
       "  : ***LOW MEMORY*** less than 10% of the memory on this node is available for use. This can cause unexpected crashes. Consider reducing the memory used by your application or reducing the Ray object store size by setting `object_store_memory` when calling `ray.init`.\n",
       "  \n",
       "  \n",
       "</div>\n",
       "<style>\n",
       ".messages {\n",
       "  color: var(--jp-ui-font-color1);\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "  padding-left: 1em;\n",
       "  overflow-y: auto;\n",
       "}\n",
       ".messages h3 {\n",
       "  font-weight: bold;\n",
       "}\n",
       ".vDivider {\n",
       "  border-left-width: var(--jp-border-width);\n",
       "  border-left-color: var(--jp-border-color0);\n",
       "  border-left-style: solid;\n",
       "  margin: 0.5em 1em 0.5em 1em;\n",
       "}\n",
       "</style>\n",
       "\n",
       "  </div>\n",
       "  <div class=\"hDivider\"></div>\n",
       "  <div class=\"trialStatus\">\n",
       "    <h3>Trial Status</h3>\n",
       "    <table>\n",
       "<thead>\n",
       "<tr><th>Trial name         </th><th>status    </th><th>loc            </th><th style=\"text-align: right;\">  batch_size</th><th style=\"text-align: right;\">  hidden_layer_1</th><th style=\"text-align: right;\">  hidden_layer_2</th><th style=\"text-align: right;\">         lr</th><th style=\"text-align: right;\">  iter</th><th style=\"text-align: right;\">  total time (s)</th><th style=\"text-align: right;\">    loss</th><th style=\"text-align: right;\">  accuracy</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>trainer_8e1f6_00000</td><td>TERMINATED</td><td>127.0.0.1:25814</td><td style=\"text-align: right;\">          48</td><td style=\"text-align: right;\">             512</td><td style=\"text-align: right;\">              64</td><td style=\"text-align: right;\">0.0131781  </td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         97.118 </td><td style=\"text-align: right;\">0.538548</td><td style=\"text-align: right;\">    0.9109</td></tr>\n",
       "<tr><td>trainer_8e1f6_00001</td><td>TERMINATED</td><td>127.0.0.1:25815</td><td style=\"text-align: right;\">          64</td><td style=\"text-align: right;\">             512</td><td style=\"text-align: right;\">              64</td><td style=\"text-align: right;\">0.00214634 </td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         91.3011</td><td style=\"text-align: right;\">1.47806 </td><td style=\"text-align: right;\">    0.8061</td></tr>\n",
       "<tr><td>trainer_8e1f6_00002</td><td>TERMINATED</td><td>127.0.0.1:25816</td><td style=\"text-align: right;\">          32</td><td style=\"text-align: right;\">             512</td><td style=\"text-align: right;\">              64</td><td style=\"text-align: right;\">0.00129984 </td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">        103.354 </td><td style=\"text-align: right;\">1.40495 </td><td style=\"text-align: right;\">    0.8321</td></tr>\n",
       "<tr><td>trainer_8e1f6_00003</td><td>TERMINATED</td><td>127.0.0.1:25817</td><td style=\"text-align: right;\">          32</td><td style=\"text-align: right;\">             256</td><td style=\"text-align: right;\">             128</td><td style=\"text-align: right;\">0.000242174</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         90.0152</td><td style=\"text-align: right;\">2.16802 </td><td style=\"text-align: right;\">    0.6958</td></tr>\n",
       "<tr><td>trainer_8e1f6_00004</td><td>TERMINATED</td><td>127.0.0.1:25818</td><td style=\"text-align: right;\">          48</td><td style=\"text-align: right;\">             512</td><td style=\"text-align: right;\">              64</td><td style=\"text-align: right;\">0.0013529  </td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         97.2941</td><td style=\"text-align: right;\">1.5946  </td><td style=\"text-align: right;\">    0.8054</td></tr>\n",
       "<tr><td>trainer_8e1f6_00005</td><td>TERMINATED</td><td>127.0.0.1:25819</td><td style=\"text-align: right;\">          32</td><td style=\"text-align: right;\">             512</td><td style=\"text-align: right;\">              64</td><td style=\"text-align: right;\">0.012783   </td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">        103.176 </td><td style=\"text-align: right;\">0.482466</td><td style=\"text-align: right;\">    0.9141</td></tr>\n",
       "<tr><td>trainer_8e1f6_00006</td><td>TERMINATED</td><td>127.0.0.1:25820</td><td style=\"text-align: right;\">          48</td><td style=\"text-align: right;\">             512</td><td style=\"text-align: right;\">             128</td><td style=\"text-align: right;\">0.000110933</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         99.5073</td><td style=\"text-align: right;\">2.24629 </td><td style=\"text-align: right;\">    0.5104</td></tr>\n",
       "<tr><td>trainer_8e1f6_00007</td><td>TERMINATED</td><td>127.0.0.1:25821</td><td style=\"text-align: right;\">          48</td><td style=\"text-align: right;\">             512</td><td style=\"text-align: right;\">              64</td><td style=\"text-align: right;\">0.00190065 </td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         97.4514</td><td style=\"text-align: right;\">1.34412 </td><td style=\"text-align: right;\">    0.8383</td></tr>\n",
       "<tr><td>trainer_8e1f6_00008</td><td>TERMINATED</td><td>127.0.0.1:25817</td><td style=\"text-align: right;\">          64</td><td style=\"text-align: right;\">             256</td><td style=\"text-align: right;\">             128</td><td style=\"text-align: right;\">0.000187373</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         40.9069</td><td style=\"text-align: right;\">2.25873 </td><td style=\"text-align: right;\">    0.415 </td></tr>\n",
       "<tr><td>trainer_8e1f6_00009</td><td>TERMINATED</td><td>127.0.0.1:25815</td><td style=\"text-align: right;\">          32</td><td style=\"text-align: right;\">             512</td><td style=\"text-align: right;\">             128</td><td style=\"text-align: right;\">0.0127463  </td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">         71.4595</td><td style=\"text-align: right;\">0.468081</td><td style=\"text-align: right;\">    0.917 </td></tr>\n",
       "</tbody>\n",
       "</table>\n",
       "  </div>\n",
       "</div>\n",
       "<style>\n",
       ".tuneStatus {\n",
       "  color: var(--jp-ui-font-color1);\n",
       "}\n",
       ".tuneStatus .systemInfo {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       ".tuneStatus td {\n",
       "  white-space: nowrap;\n",
       "}\n",
       ".tuneStatus .trialStatus {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       ".tuneStatus h3 {\n",
       "  font-weight: bold;\n",
       "}\n",
       ".tuneStatus .hDivider {\n",
       "  border-bottom-width: var(--jp-border-width);\n",
       "  border-bottom-color: var(--jp-border-color0);\n",
       "  border-bottom-style: solid;\n",
       "}\n",
       ".tuneStatus .vDivider {\n",
       "  border-left-width: var(--jp-border-width);\n",
       "  border-left-color: var(--jp-border-color0);\n",
       "  border-left-style: solid;\n",
       "  margin: 0.5em 1em 0.5em 1em;\n",
       "}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div class=\"trialProgress\">\n",
       "  <h3>Trial Progress</h3>\n",
       "  <table>\n",
       "<thead>\n",
       "<tr><th>Trial name         </th><th style=\"text-align: right;\">  accuracy</th><th style=\"text-align: right;\">    loss</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>trainer_8e1f6_00000</td><td style=\"text-align: right;\">    0.9109</td><td style=\"text-align: right;\">0.538548</td></tr>\n",
       "<tr><td>trainer_8e1f6_00001</td><td style=\"text-align: right;\">    0.8061</td><td style=\"text-align: right;\">1.47806 </td></tr>\n",
       "<tr><td>trainer_8e1f6_00002</td><td style=\"text-align: right;\">    0.8321</td><td style=\"text-align: right;\">1.40495 </td></tr>\n",
       "<tr><td>trainer_8e1f6_00003</td><td style=\"text-align: right;\">    0.6958</td><td style=\"text-align: right;\">2.16802 </td></tr>\n",
       "<tr><td>trainer_8e1f6_00004</td><td style=\"text-align: right;\">    0.8054</td><td style=\"text-align: right;\">1.5946  </td></tr>\n",
       "<tr><td>trainer_8e1f6_00005</td><td style=\"text-align: right;\">    0.9141</td><td style=\"text-align: right;\">0.482466</td></tr>\n",
       "<tr><td>trainer_8e1f6_00006</td><td style=\"text-align: right;\">    0.5104</td><td style=\"text-align: right;\">2.24629 </td></tr>\n",
       "<tr><td>trainer_8e1f6_00007</td><td style=\"text-align: right;\">    0.8383</td><td style=\"text-align: right;\">1.34412 </td></tr>\n",
       "<tr><td>trainer_8e1f6_00008</td><td style=\"text-align: right;\">    0.415 </td><td style=\"text-align: right;\">2.25873 </td></tr>\n",
       "<tr><td>trainer_8e1f6_00009</td><td style=\"text-align: right;\">    0.917 </td><td style=\"text-align: right;\">0.468081</td></tr>\n",
       "</tbody>\n",
       "</table>\n",
       "</div>\n",
       "<style>\n",
       ".trialProgress {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "  color: var(--jp-ui-font-color1);\n",
       "}\n",
       ".trialProgress h3 {\n",
       "  font-weight: bold;\n",
       "}\n",
       ".trialProgress td {\n",
       "  white-space: nowrap;\n",
       "}\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-10-18 21:42:44,171\tINFO tune.py:1143 -- Total run time: 167.02 seconds (166.98 seconds for the tuning loop).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best trial config: {'hidden_layer_1': 512, 'hidden_layer_2': 128, 'lr': 0.012746297610971511, 'batch_size': 32}\n",
      "Best trial final validation loss:  0.46808149814605715\n",
      "Best trial final validation loss:  0.917\n"
     ]
    }
   ],
   "source": [
    "result = tune.run(\n",
    "        partial(trainer,epochs=5, data_dir='/Users/atifadib/Deep_Learning_with_PyTorch/chapter3/output'),\n",
    "        resources_per_trial={\"cpu\": 1, \"gpus_per_trial\": 0},\n",
    "        config=config,\n",
    "        num_samples=num_samples,\n",
    "        scheduler=scheduler,\n",
    "    )\n",
    "\n",
    "best_trial = result.get_best_trial(\"loss\", \"min\", \"last\")\n",
    "print(f\"Best trial config: {best_trial.config}\")\n",
    "print(f\"Best trial final validation loss:  {best_trial.last_result['loss']}\")\n",
    "print(f\"Best trial final validation loss:  {best_trial.last_result['accuracy']}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3798fe4d",
   "metadata": {},
   "source": [
    "#### Saving and Reloading a Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "4f1bac40",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(), 'model.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "2f62cd20",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_new = NeuralNet(512, 128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "024c5690",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_new.load_state_dict(torch.load('model.pth'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eebdeba2",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch",
   "language": "python",
   "name": "torch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
